00000800:00000200:2.1F:1734015831.219199:0:1967:0:(socklnd.c:225:ksocknal_find_peer_locked()) got peer_ni [ffff8800b6287780] -> 12345-192.168.201.104@tcp (2) 00000800:00000200:0.1F:1734015831.222729:0:1956:0:(socklnd.c:409:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:0.1:1734015831.222742:0:1956:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.104@tcp ip 192.168.201.104:988 00000800:00000200:0.1:1734015831.222745:0:1956:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8800b5b6af00 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.1:1734015831.224989:0:1956:0:(socklnd.c:409:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:0.1:1734015831.226532:0:1956:0:(socklnd.c:409:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000080:00200000:1.0:1734016138.423881:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2461696 00000080:00200000:1.0:1734016138.423882:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 601 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.423888:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2461696, len 4096, copied 4096 00000080:00200000:1.0:1734016138.423890:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.423891:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990200 2fffff00000805 3:0 ffff88012e3d0af8 601 lru 00000080:00200000:1.0:1734016138.423893:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0b58 601: 1< 0 - - > 2< 2461696 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487337984 - | - - + - > 5< - - + - | 0 - | 601 - -> 00000080:00200000:1.0:1734016138.423894:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0af8 00000080:00200000:1.0:1734016138.423895:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.423895:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.423898:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423901:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423904:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423907:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.423911:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.423913:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2465792 00000080:00200000:1.0:1734016138.423914:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 602 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.423920:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2465792, len 4096, copied 4096 00000080:00200000:1.0:1734016138.423921:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.423922:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990240 2fffff00000805 3:0 ffff88012e3d0a20 602 lru 00000080:00200000:1.0:1734016138.423925:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0a80 602: 1< 0 - - > 2< 2465792 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487333888 - | - - + - > 5< - - + - | 0 - | 602 - -> 00000080:00200000:1.0:1734016138.423926:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0a20 00000080:00200000:1.0:1734016138.423926:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.423927:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.423929:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423933:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423936:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423939:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.423943:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.423944:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2469888 00000080:00200000:1.0:1734016138.423945:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 603 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.423951:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2469888, len 4096, copied 4096 00000080:00200000:1.0:1734016138.423952:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.423953:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990280 2fffff00000805 3:0 ffff88012e3d0948 603 lru 00000080:00200000:1.0:1734016138.423956:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d09a8 603: 1< 0 - - > 2< 2469888 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487329792 - | - - + - > 5< - - + - | 0 - | 603 - -> 00000080:00200000:1.0:1734016138.423956:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0948 00000080:00200000:1.0:1734016138.423957:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.423958:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.423960:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423963:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423966:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423970:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.423974:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.423975:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2473984 00000080:00200000:1.0:1734016138.423976:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 604 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.423982:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2473984, len 4096, copied 4096 00000080:00200000:1.0:1734016138.423983:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.423985:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049902c0 2fffff00000805 3:0 ffff88012e3d0870 604 lru 00000080:00200000:1.0:1734016138.423987:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d08d0 604: 1< 0 - - > 2< 2473984 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487325696 - | - - + - > 5< - - + - | 0 - | 604 - -> 00000080:00200000:1.0:1734016138.423988:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0870 00000080:00200000:1.0:1734016138.423988:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.423989:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.423991:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423994:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.423997:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424000:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424004:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424005:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2478080 00000080:00200000:1.0:1734016138.424006:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 605 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424012:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2478080, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424014:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424015:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990300 2fffff00000805 3:0 ffff88012e3d0798 605 lru 00000080:00200000:1.0:1734016138.424017:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d07f8 605: 1< 0 - - > 2< 2478080 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487321600 - | - - + - > 5< - - + - | 0 - | 605 - -> 00000080:00200000:1.0:1734016138.424018:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0798 00000080:00200000:1.0:1734016138.424018:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424019:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424021:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424024:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424027:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424030:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424034:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424036:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2482176 00000080:00200000:1.0:1734016138.424037:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 606 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424043:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2482176, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424044:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d06c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424045:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990340 2fffff00000805 3:0 ffff88012e3d06c0 606 lru 00000080:00200000:1.0:1734016138.424048:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0720 606: 1< 0 - - > 2< 2482176 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487317504 - | - - + - > 5< - - + - | 0 - | 606 - -> 00000080:00200000:1.0:1734016138.424049:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d06c0 00000080:00200000:1.0:1734016138.424049:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424049:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424053:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424057:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424060:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424063:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424066:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424068:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2486272 00000080:00200000:1.0:1734016138.424069:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 607 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424075:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2486272, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424076:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d05e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424077:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990380 2fffff00000805 3:0 ffff88012e3d05e8 607 lru 00000080:00200000:1.0:1734016138.424080:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0648 607: 1< 0 - - > 2< 2486272 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487313408 - | - - + - > 5< - - + - | 0 - | 607 - -> 00000080:00200000:1.0:1734016138.424081:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d05e8 00000080:00200000:1.0:1734016138.424081:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424082:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424084:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424087:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424091:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424095:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424099:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424100:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2490368 00000080:00200000:1.0:1734016138.424101:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 608 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424107:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2490368, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424108:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424109:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049903c0 2fffff00000805 3:0 ffff88012e3d0510 608 lru 00000080:00200000:1.0:1734016138.424112:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0570 608: 1< 0 - - > 2< 2490368 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487309312 - | - - + - > 5< - - + - | 0 - | 608 - -> 00000080:00200000:1.0:1734016138.424113:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0510 00000080:00200000:1.0:1734016138.424113:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424124:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424128:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424131:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424134:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424137:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424142:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424144:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2494464 00000080:00200000:1.0:1734016138.424146:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 609 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424154:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2494464, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424155:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424157:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990400 2fffff00000805 3:0 ffff88012e3d0438 609 lru 00000080:00200000:1.0:1734016138.424159:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0498 609: 1< 0 - - > 2< 2494464 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487305216 - | - - + - > 5< - - + - | 0 - | 609 - -> 00000080:00200000:1.0:1734016138.424160:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0438 00000080:00200000:1.0:1734016138.424161:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424161:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424164:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424167:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424170:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424173:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424176:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424178:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2498560 00000080:00200000:1.0:1734016138.424179:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 610 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424185:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2498560, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424186:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424187:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990440 2fffff00000805 3:0 ffff88012e3d0360 610 lru 00000080:00200000:1.0:1734016138.424190:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d03c0 610: 1< 0 - - > 2< 2498560 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487301120 - | - - + - > 5< - - + - | 0 - | 610 - -> 00000080:00200000:1.0:1734016138.424191:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0360 00000080:00200000:1.0:1734016138.424191:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424191:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424194:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424198:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424201:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424204:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424208:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424209:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2502656 00000080:00200000:1.0:1734016138.424210:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 611 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424217:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2502656, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424218:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424220:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990480 2fffff00000825 2:0 ffff88012e3d0288 611 lru 00000080:00200000:1.0:1734016138.424222:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d02e8 611: 1< 0 - - > 2< 2502656 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487297024 - | - - + - > 5< - - + - | 0 - | 611 - -> 00000080:00200000:1.0:1734016138.424223:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0288 00000080:00200000:1.0:1734016138.424223:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424224:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424226:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424229:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424233:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424236:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424241:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424242:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2506752 00000080:00200000:1.0:1734016138.424243:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 612 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424249:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2506752, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424250:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d01b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424251:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049904c0 2fffff00000805 3:0 ffff88012e3d01b0 612 lru 00000080:00200000:1.0:1734016138.424254:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0210 612: 1< 0 - - > 2< 2506752 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487292928 - | - - + - > 5< - - + - | 0 - | 612 - -> 00000080:00200000:1.0:1734016138.424255:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d01b0 00000080:00200000:1.0:1734016138.424255:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424255:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424258:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424261:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424264:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424268:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424272:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424273:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2510848 00000080:00200000:1.0:1734016138.424274:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 613 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424280:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2510848, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424282:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d00d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424283:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990500 2fffff00000805 3:0 ffff88012e3d00d8 613 lru 00000080:00200000:1.0:1734016138.424285:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0138 613: 1< 0 - - > 2< 2510848 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487288832 - | - - + - > 5< - - + - | 0 - | 613 - -> 00000080:00200000:1.0:1734016138.424286:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d00d8 00000080:00200000:1.0:1734016138.424286:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424287:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424289:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424293:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424295:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424298:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424302:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424304:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2514944 00000080:00200000:1.0:1734016138.424305:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 614 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424311:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2514944, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424312:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012e3d0000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424313:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990540 2fffff00000805 3:0 ffff88012e3d0000 614 lru 00000080:00200000:1.0:1734016138.424316:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012e3d0060 614: 1< 0 - - > 2< 2514944 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487284736 - | - - + - > 5< - - + - | 0 - | 614 - -> 00000080:00200000:1.0:1734016138.424316:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012e3d0000 00000080:00200000:1.0:1734016138.424317:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424317:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424320:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424323:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424326:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424329:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424333:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424334:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2519040 00000080:00200000:1.0:1734016138.424335:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 615 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424343:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2519040, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424344:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424346:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990580 2fffff00000805 3:0 ffff88012bda7e58 615 lru 00000080:00200000:1.0:1734016138.424348:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7eb8 615: 1< 0 - - > 2< 2519040 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487280640 - | - - + - > 5< - - + - | 0 - | 615 - -> 00000080:00200000:1.0:1734016138.424349:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7e58 00000080:00200000:1.0:1734016138.424349:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424350:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424352:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424355:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424359:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424363:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424367:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424368:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2523136 00000080:00200000:1.0:1734016138.424369:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 616 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424375:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2523136, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424376:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424377:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049905c0 2fffff00000805 3:0 ffff88012bda7d80 616 lru 00000080:00200000:1.0:1734016138.424380:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7de0 616: 1< 0 - - > 2< 2523136 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487276544 - | - - + - > 5< - - + - | 0 - | 616 - -> 00000080:00200000:1.0:1734016138.424380:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7d80 00000080:00200000:1.0:1734016138.424381:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424381:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424384:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424387:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424390:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424393:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424397:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424398:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2527232 00000080:00200000:1.0:1734016138.424399:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 617 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424406:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2527232, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424408:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424409:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990600 2fffff00000805 3:0 ffff88012bda7ca8 617 lru 00000080:00200000:1.0:1734016138.424411:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7d08 617: 1< 0 - - > 2< 2527232 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487272448 - | - - + - > 5< - - + - | 0 - | 617 - -> 00000080:00200000:1.0:1734016138.424412:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7ca8 00000080:00200000:1.0:1734016138.424413:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424413:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424416:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424419:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424421:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424424:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424428:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424430:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2531328 00000080:00200000:1.0:1734016138.424431:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 618 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424436:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2531328, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424438:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424439:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990640 2fffff00000805 3:0 ffff88012bda7bd0 618 lru 00000080:00200000:1.0:1734016138.424441:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7c30 618: 1< 0 - - > 2< 2531328 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487268352 - | - - + - > 5< - - + - | 0 - | 618 - -> 00000080:00200000:1.0:1734016138.424442:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7bd0 00000080:00200000:1.0:1734016138.424442:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424443:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424446:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424449:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424452:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424455:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424459:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424460:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2535424 00000080:00200000:1.0:1734016138.424461:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 619 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424467:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2535424, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424468:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424469:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990680 2fffff00000805 3:0 ffff88012bda7af8 619 lru 00000080:00200000:1.0:1734016138.424472:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7b58 619: 1< 0 - - > 2< 2535424 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487264256 - | - - + - > 5< - - + - | 0 - | 619 - -> 00000080:00200000:1.0:1734016138.424472:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7af8 00000080:00200000:1.0:1734016138.424473:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424473:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424476:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424479:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424482:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424485:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424489:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424490:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2539520 00000080:00200000:1.0:1734016138.424491:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 620 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424497:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2539520, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424499:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424500:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049906c0 2fffff00000805 3:0 ffff88012bda7a20 620 lru 00000080:00200000:1.0:1734016138.424502:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7a80 620: 1< 0 - - > 2< 2539520 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487260160 - | - - + - > 5< - - + - | 0 - | 620 - -> 00000080:00200000:1.0:1734016138.424503:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7a20 00000080:00200000:1.0:1734016138.424503:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424504:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424506:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424509:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424512:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424515:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424519:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424521:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2543616 00000080:00200000:1.0:1734016138.424523:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 621 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424528:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2543616, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424530:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424531:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990700 2fffff00000805 3:0 ffff88012bda7948 621 lru 00000080:00200000:1.0:1734016138.424534:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda79a8 621: 1< 0 - - > 2< 2543616 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487256064 - | - - + - > 5< - - + - | 0 - | 621 - -> 00000080:00200000:1.0:1734016138.424534:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7948 00000080:00200000:1.0:1734016138.424535:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424535:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424538:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424541:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424544:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424547:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424550:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424552:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2547712 00000080:00200000:1.0:1734016138.424553:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 622 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424559:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2547712, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424560:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424561:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990740 2fffff00000805 3:0 ffff88012bda7870 622 lru 00000080:00200000:1.0:1734016138.424564:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda78d0 622: 1< 0 - - > 2< 2547712 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487251968 - | - - + - > 5< - - + - | 0 - | 622 - -> 00000080:00200000:1.0:1734016138.424564:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7870 00000080:00200000:1.0:1734016138.424565:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424565:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424568:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424571:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424574:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424577:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424581:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424582:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2551808 00000080:00200000:1.0:1734016138.424583:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 623 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424590:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2551808, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424591:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424592:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990780 2fffff00000805 3:0 ffff88012bda7798 623 lru 00000080:00200000:1.0:1734016138.424595:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda77f8 623: 1< 0 - - > 2< 2551808 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487247872 - | - - + - > 5< - - + - | 0 - | 623 - -> 00000080:00200000:1.0:1734016138.424595:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7798 00000080:00200000:1.0:1734016138.424596:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424596:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424599:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424602:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424605:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424609:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424614:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424615:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2555904 00000080:00200000:1.0:1734016138.424616:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 624 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424626:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2555904, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424627:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda76c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424628:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049907c0 2fffff00000805 3:0 ffff88012bda76c0 624 lru 00000080:00200000:1.0:1734016138.424631:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7720 624: 1< 0 - - > 2< 2555904 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487243776 - | - - + - > 5< - - + - | 0 - | 624 - -> 00000080:00200000:1.0:1734016138.424632:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda76c0 00000080:00200000:1.0:1734016138.424632:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424633:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424635:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424639:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424642:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424645:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424648:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424650:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2560000 00000080:00200000:1.0:1734016138.424651:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 625 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424659:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2560000, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424660:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda75e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424661:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990800 2fffff00000825 2:0 ffff88012bda75e8 625 lru 00000080:00200000:1.0:1734016138.424663:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7648 625: 1< 0 - - > 2< 2560000 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487239680 - | - - + - > 5< - - + - | 0 - | 625 - -> 00000080:00200000:1.0:1734016138.424664:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda75e8 00000080:00200000:1.0:1734016138.424665:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424665:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424668:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424671:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424674:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424677:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424681:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424682:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2564096 00000080:00200000:1.0:1734016138.424683:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 626 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424704:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2564096, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424706:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424707:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990840 2fffff00000805 3:0 ffff88012bda7510 626 lru 00000080:00200000:1.0:1734016138.424710:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7570 626: 1< 0 - - > 2< 2564096 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487235584 - | - - + - > 5< - - + - | 0 - | 626 - -> 00000080:00200000:1.0:1734016138.424711:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7510 00000080:00200000:1.0:1734016138.424711:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424712:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424716:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424720:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424723:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424726:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424730:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424732:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2568192 00000080:00200000:1.0:1734016138.424733:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 627 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424740:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2568192, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424742:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424743:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990880 2fffff00000805 3:0 ffff88012bda7438 627 lru 00000080:00200000:1.0:1734016138.424745:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7498 627: 1< 0 - - > 2< 2568192 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487231488 - | - - + - > 5< - - + - | 0 - | 627 - -> 00000080:00200000:1.0:1734016138.424746:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7438 00000080:00200000:1.0:1734016138.424746:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424747:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424749:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424753:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424757:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424761:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424766:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424767:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2572288 00000080:00200000:1.0:1734016138.424768:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 628 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424774:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2572288, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424776:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424777:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049908c0 2fffff00000805 3:0 ffff88012bda7360 628 lru 00000080:00200000:1.0:1734016138.424780:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda73c0 628: 1< 0 - - > 2< 2572288 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487227392 - | - - + - > 5< - - + - | 0 - | 628 - -> 00000080:00200000:1.0:1734016138.424781:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7360 00000080:00200000:1.0:1734016138.424781:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424782:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424784:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424787:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424790:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424793:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424798:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424800:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2576384 00000080:00200000:1.0:1734016138.424801:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 629 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424807:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2576384, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424809:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424810:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990900 2fffff00000805 3:0 ffff88012bda7288 629 lru 00000080:00200000:1.0:1734016138.424813:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda72e8 629: 1< 0 - - > 2< 2576384 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487223296 - | - - + - > 5< - - + - | 0 - | 629 - -> 00000080:00200000:1.0:1734016138.424814:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7288 00000080:00200000:1.0:1734016138.424814:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424814:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424817:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424820:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424823:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424826:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424830:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424831:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2580480 00000080:00200000:1.0:1734016138.424832:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 630 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424838:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2580480, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424839:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda71b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424840:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990940 2fffff00000805 3:0 ffff88012bda71b0 630 lru 00000080:00200000:1.0:1734016138.424843:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7210 630: 1< 0 - - > 2< 2580480 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487219200 - | - - + - > 5< - - + - | 0 - | 630 - -> 00000080:00200000:1.0:1734016138.424843:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda71b0 00000080:00200000:1.0:1734016138.424844:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424844:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424847:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424850:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424853:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424856:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424860:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424861:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2584576 00000080:00200000:1.0:1734016138.424862:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 631 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424868:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2584576, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424870:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda70d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424871:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990980 2fffff00000805 3:0 ffff88012bda70d8 631 lru 00000080:00200000:1.0:1734016138.424873:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7138 631: 1< 0 - - > 2< 2584576 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487215104 - | - - + - > 5< - - + - | 0 - | 631 - -> 00000080:00200000:1.0:1734016138.424874:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda70d8 00000080:00200000:1.0:1734016138.424874:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424875:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424878:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424881:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424883:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424887:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424892:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424893:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2588672 00000080:00200000:1.0:1734016138.424894:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 632 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424900:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2588672, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424902:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff88012bda7000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424903:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049909c0 2fffff00000805 3:0 ffff88012bda7000 632 lru 00000080:00200000:1.0:1734016138.424905:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff88012bda7060 632: 1< 0 - - > 2< 2588672 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487211008 - | - - + - > 5< - - + - | 0 - | 632 - -> 00000080:00200000:1.0:1734016138.424906:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff88012bda7000 00000080:00200000:1.0:1734016138.424906:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424907:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424909:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424913:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424917:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424920:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424923:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424925:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2592768 00000080:00200000:1.0:1734016138.424926:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 633 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424933:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2592768, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424935:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758de58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424936:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990a00 2fffff00000805 3:0 ffff8800a758de58 633 lru 00000080:00200000:1.0:1734016138.424938:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758deb8 633: 1< 0 - - > 2< 2592768 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487206912 - | - - + - > 5< - - + - | 0 - | 633 - -> 00000080:00200000:1.0:1734016138.424939:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758de58 00000080:00200000:1.0:1734016138.424940:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424940:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424943:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424946:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424949:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424952:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424956:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424957:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2596864 00000080:00200000:1.0:1734016138.424958:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 634 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424964:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2596864, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424965:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758dd80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424966:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990a40 2fffff00000805 3:0 ffff8800a758dd80 634 lru 00000080:00200000:1.0:1734016138.424969:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758dde0 634: 1< 0 - - > 2< 2596864 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487202816 - | - - + - > 5< - - + - | 0 - | 634 - -> 00000080:00200000:1.0:1734016138.424970:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758dd80 00000080:00200000:1.0:1734016138.424970:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.424971:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.424973:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424976:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424979:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.424982:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.424986:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.424988:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2600960 00000080:00200000:1.0:1734016138.424989:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 635 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.424995:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2600960, len 4096, copied 4096 00000080:00200000:1.0:1734016138.424996:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758dca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.424997:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990a80 2fffff00000805 3:0 ffff8800a758dca8 635 lru 00000080:00200000:1.0:1734016138.425000:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758dd08 635: 1< 0 - - > 2< 2600960 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487198720 - | - - + - > 5< - - + - | 0 - | 635 - -> 00000080:00200000:1.0:1734016138.425000:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758dca8 00000080:00200000:1.0:1734016138.425001:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425001:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425004:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425007:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425010:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425014:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425018:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425019:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2605056 00000080:00200000:1.0:1734016138.425021:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 636 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425026:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2605056, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425028:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758dbd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425029:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990ac0 2fffff00000805 3:0 ffff8800a758dbd0 636 lru 00000080:00200000:1.0:1734016138.425031:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758dc30 636: 1< 0 - - > 2< 2605056 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487194624 - | - - + - > 5< - - + - | 0 - | 636 - -> 00000080:00200000:1.0:1734016138.425032:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758dbd0 00000080:00200000:1.0:1734016138.425033:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425033:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425036:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425040:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425043:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425046:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425049:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425051:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2609152 00000080:00200000:1.0:1734016138.425052:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 637 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425058:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2609152, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425059:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758daf8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425061:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990b00 2fffff00000805 3:0 ffff8800a758daf8 637 lru 00000080:00200000:1.0:1734016138.425063:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758db58 637: 1< 0 - - > 2< 2609152 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487190528 - | - - + - > 5< - - + - | 0 - | 637 - -> 00000080:00200000:1.0:1734016138.425064:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758daf8 00000080:00200000:1.0:1734016138.425064:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425065:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425067:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425070:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425073:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425076:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425080:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425081:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2613248 00000080:00200000:1.0:1734016138.425082:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 638 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425088:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2613248, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425089:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758da20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425091:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990b40 2fffff00000805 3:0 ffff8800a758da20 638 lru 00000080:00200000:1.0:1734016138.425093:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758da80 638: 1< 0 - - > 2< 2613248 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487186432 - | - - + - > 5< - - + - | 0 - | 638 - -> 00000080:00200000:1.0:1734016138.425094:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758da20 00000080:00200000:1.0:1734016138.425094:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425095:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425097:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425101:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425104:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425107:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425111:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425112:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2617344 00000080:00200000:1.0:1734016138.425113:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 639 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425135:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2617344, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425137:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425138:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990b80 2fffff00000825 2:0 ffff8800a758d948 639 lru 00000080:00200000:1.0:1734016138.425141:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d9a8 639: 1< 0 - - > 2< 2617344 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487182336 - | - - + - > 5< - - + - | 0 - | 639 - -> 00000080:00200000:1.0:1734016138.425142:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d948 00000080:00200000:1.0:1734016138.425142:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425143:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425146:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425149:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425153:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425157:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425162:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425163:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2621440 00000080:00200000:1.0:1734016138.425164:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 640 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425171:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2621440, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425173:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425174:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990bc0 2fffff00000805 3:0 ffff8800a758d870 640 lru 00000080:00200000:1.0:1734016138.425177:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d8d0 640: 1< 0 - - > 2< 2621440 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487178240 - | - - + - > 5< - - + - | 0 - | 640 - -> 00000080:00200000:1.0:1734016138.425178:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d870 00000080:00200000:1.0:1734016138.425178:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425179:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425182:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425185:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425188:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425191:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425195:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425196:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2625536 00000080:00200000:1.0:1734016138.425197:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 641 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425203:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2625536, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425205:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425206:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990c00 2fffff00000805 3:0 ffff8800a758d798 641 lru 00000080:00200000:1.0:1734016138.425209:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d7f8 641: 1< 0 - - > 2< 2625536 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487174144 - | - - + - > 5< - - + - | 0 - | 641 - -> 00000080:00200000:1.0:1734016138.425209:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d798 00000080:00200000:1.0:1734016138.425210:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425210:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425213:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425216:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425219:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425222:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425226:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425227:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2629632 00000080:00200000:1.0:1734016138.425228:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 642 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425235:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2629632, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425236:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d6c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425237:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990c40 2fffff00000805 3:0 ffff8800a758d6c0 642 lru 00000080:00200000:1.0:1734016138.425244:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d720 642: 1< 0 - - > 2< 2629632 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487170048 - | - - + - > 5< - - + - | 0 - | 642 - -> 00000080:00200000:1.0:1734016138.425245:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d6c0 00000080:00200000:1.0:1734016138.425245:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425246:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425249:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425253:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425256:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425259:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425263:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425264:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2633728 00000080:00200000:1.0:1734016138.425265:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 643 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425271:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2633728, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425273:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d5e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425274:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990c80 2fffff00000805 3:0 ffff8800a758d5e8 643 lru 00000080:00200000:1.0:1734016138.425276:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d648 643: 1< 0 - - > 2< 2633728 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487165952 - | - - + - > 5< - - + - | 0 - | 643 - -> 00000080:00200000:1.0:1734016138.425277:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d5e8 00000080:00200000:1.0:1734016138.425277:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425278:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425280:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425284:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425287:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425291:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425296:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425297:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2637824 00000080:00200000:1.0:1734016138.425298:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 644 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425304:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2637824, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425305:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425306:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990cc0 2fffff00000805 3:0 ffff8800a758d510 644 lru 00000080:00200000:1.0:1734016138.425309:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d570 644: 1< 0 - - > 2< 2637824 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487161856 - | - - + - > 5< - - + - | 0 - | 644 - -> 00000080:00200000:1.0:1734016138.425310:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d510 00000080:00200000:1.0:1734016138.425310:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425311:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425313:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425316:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425319:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425322:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425326:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425328:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2641920 00000080:00200000:1.0:1734016138.425329:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 645 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425335:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2641920, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425336:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425337:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990d00 2fffff00000805 3:0 ffff8800a758d438 645 lru 00000080:00200000:1.0:1734016138.425340:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d498 645: 1< 0 - - > 2< 2641920 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487157760 - | - - + - > 5< - - + - | 0 - | 645 - -> 00000080:00200000:1.0:1734016138.425341:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d438 00000080:00200000:1.0:1734016138.425341:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425342:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425345:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425348:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425350:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425353:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425358:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425360:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2646016 00000080:00200000:1.0:1734016138.425361:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 646 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425367:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2646016, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425369:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425370:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990d40 2fffff00000805 3:0 ffff8800a758d360 646 lru 00000080:00200000:1.0:1734016138.425372:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d3c0 646: 1< 0 - - > 2< 2646016 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487153664 - | - - + - > 5< - - + - | 0 - | 646 - -> 00000080:00200000:1.0:1734016138.425373:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d360 00000080:00200000:1.0:1734016138.425374:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425374:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425377:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425381:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425384:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425387:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425390:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425392:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2650112 00000080:00200000:1.0:1734016138.425393:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 647 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425399:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2650112, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425400:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425401:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990d80 2fffff00000805 3:0 ffff8800a758d288 647 lru 00000080:00200000:1.0:1734016138.425403:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d2e8 647: 1< 0 - - > 2< 2650112 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487149568 - | - - + - > 5< - - + - | 0 - | 647 - -> 00000080:00200000:1.0:1734016138.425404:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d288 00000080:00200000:1.0:1734016138.425405:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425405:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425408:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425411:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425414:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425417:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425421:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425422:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2654208 00000080:00200000:1.0:1734016138.425423:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 648 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425429:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2654208, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425430:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d1b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425431:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990dc0 2fffff00000805 3:0 ffff8800a758d1b0 648 lru 00000080:00200000:1.0:1734016138.425434:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d210 648: 1< 0 - - > 2< 2654208 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487145472 - | - - + - > 5< - - + - | 0 - | 648 - -> 00000080:00200000:1.0:1734016138.425435:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d1b0 00000080:00200000:1.0:1734016138.425435:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425436:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425438:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425441:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425444:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425447:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425451:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425452:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2658304 00000080:00200000:1.0:1734016138.425454:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 649 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425460:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2658304, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425461:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d0d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425463:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990e00 2fffff00000805 3:0 ffff8800a758d0d8 649 lru 00000080:00200000:1.0:1734016138.425465:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d138 649: 1< 0 - - > 2< 2658304 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487141376 - | - - + - > 5< - - + - | 0 - | 649 - -> 00000080:00200000:1.0:1734016138.425466:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d0d8 00000080:00200000:1.0:1734016138.425466:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425467:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425469:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425472:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425475:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425478:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425482:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425484:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2662400 00000080:00200000:1.0:1734016138.425485:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 650 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425490:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2662400, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425492:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a758d000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425493:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990e40 2fffff00000805 3:0 ffff8800a758d000 650 lru 00000080:00200000:1.0:1734016138.425495:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a758d060 650: 1< 0 - - > 2< 2662400 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487137280 - | - - + - > 5< - - + - | 0 - | 650 - -> 00000080:00200000:1.0:1734016138.425496:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a758d000 00000080:00200000:1.0:1734016138.425496:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425497:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425500:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425503:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425506:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425509:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425512:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425514:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2666496 00000080:00200000:1.0:1734016138.425515:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 651 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425521:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2666496, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425523:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862fe58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425524:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990e80 2fffff00000805 3:0 ffff8800a862fe58 651 lru 00000080:00200000:1.0:1734016138.425526:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862feb8 651: 1< 0 - - > 2< 2666496 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487133184 - | - - + - > 5< - - + - | 0 - | 651 - -> 00000080:00200000:1.0:1734016138.425527:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862fe58 00000080:00200000:1.0:1734016138.425527:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425528:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425530:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425533:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425536:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425540:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425544:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425546:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2670592 00000080:00200000:1.0:1734016138.425547:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 652 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425553:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2670592, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425554:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862fd80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425556:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990ec0 2fffff00000805 3:0 ffff8800a862fd80 652 lru 00000080:00200000:1.0:1734016138.425558:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862fde0 652: 1< 0 - - > 2< 2670592 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487129088 - | - - + - > 5< - - + - | 0 - | 652 - -> 00000080:00200000:1.0:1734016138.425559:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862fd80 00000080:00200000:1.0:1734016138.425559:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425560:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425562:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425565:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425568:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425571:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425576:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425577:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2674688 00000080:00200000:1.0:1734016138.425579:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 653 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425586:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2674688, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425587:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862fca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425589:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990f00 2fffff00000825 2:0 ffff8800a862fca8 653 lru 00000080:00200000:1.0:1734016138.425591:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862fd08 653: 1< 0 - - > 2< 2674688 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487124992 - | - - + - > 5< - - + - | 0 - | 653 - -> 00000080:00200000:1.0:1734016138.425592:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862fca8 00000080:00200000:1.0:1734016138.425592:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425593:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425595:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425599:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425601:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425604:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425608:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425610:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2678784 00000080:00200000:1.0:1734016138.425611:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 654 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425616:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2678784, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425617:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862fbd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425619:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990f40 2fffff00000805 3:0 ffff8800a862fbd0 654 lru 00000080:00200000:1.0:1734016138.425621:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862fc30 654: 1< 0 - - > 2< 2678784 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487120896 - | - - + - > 5< - - + - | 0 - | 654 - -> 00000080:00200000:1.0:1734016138.425622:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862fbd0 00000080:00200000:1.0:1734016138.425622:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425623:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425625:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425630:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425633:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425636:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425640:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425641:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2682880 00000080:00200000:1.0:1734016138.425642:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 655 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425652:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2682880, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425653:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862faf8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425655:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990f80 2fffff00000805 3:0 ffff8800a862faf8 655 lru 00000080:00200000:1.0:1734016138.425657:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862fb58 655: 1< 0 - - > 2< 2682880 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487116800 - | - - + - > 5< - - + - | 0 - | 655 - -> 00000080:00200000:1.0:1734016138.425658:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862faf8 00000080:00200000:1.0:1734016138.425658:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425659:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425661:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425664:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425668:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425671:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425675:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425677:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2686976 00000080:00200000:1.0:1734016138.425678:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 656 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425684:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2686976, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425685:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862fa20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425687:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990fc0 2fffff00000805 3:0 ffff8800a862fa20 656 lru 00000080:00200000:1.0:1734016138.425708:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862fa80 656: 1< 0 - - > 2< 2686976 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487112704 - | - - + - > 5< - - + - | 0 - | 656 - -> 00000080:00200000:1.0:1734016138.425709:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862fa20 00000080:00200000:1.0:1734016138.425710:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425710:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425714:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425717:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425720:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425724:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425728:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425730:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2691072 00000080:00200000:1.0:1734016138.425732:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 657 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425739:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2691072, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425741:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425742:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991000 2fffff00000805 3:0 ffff8800a862f948 657 lru 00000080:00200000:1.0:1734016138.425745:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f9a8 657: 1< 0 - - > 2< 2691072 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487108608 - | - - + - > 5< - - + - | 0 - | 657 - -> 00000080:00200000:1.0:1734016138.425746:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f948 00000080:00200000:1.0:1734016138.425746:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425747:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425749:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425752:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425755:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425758:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425762:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425763:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2695168 00000080:00200000:1.0:1734016138.425764:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 658 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425770:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2695168, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425772:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425773:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991040 2fffff00000805 3:0 ffff8800a862f870 658 lru 00000080:00200000:1.0:1734016138.425775:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f8d0 658: 1< 0 - - > 2< 2695168 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487104512 - | - - + - > 5< - - + - | 0 - | 658 - -> 00000080:00200000:1.0:1734016138.425776:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f870 00000080:00200000:1.0:1734016138.425776:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425777:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425780:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425784:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425787:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425790:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425794:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425795:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2699264 00000080:00200000:1.0:1734016138.425796:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 659 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425802:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2699264, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425803:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425804:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991080 2fffff00000805 3:0 ffff8800a862f798 659 lru 00000080:00200000:1.0:1734016138.425807:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f7f8 659: 1< 0 - - > 2< 2699264 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487100416 - | - - + - > 5< - - + - | 0 - | 659 - -> 00000080:00200000:1.0:1734016138.425807:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f798 00000080:00200000:1.0:1734016138.425808:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425808:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425811:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425814:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425817:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425820:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425824:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425825:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2703360 00000080:00200000:1.0:1734016138.425826:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 660 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425832:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2703360, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425834:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f6c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425835:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049910c0 2fffff00000805 3:0 ffff8800a862f6c0 660 lru 00000080:00200000:1.0:1734016138.425837:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f720 660: 1< 0 - - > 2< 2703360 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487096320 - | - - + - > 5< - - + - | 0 - | 660 - -> 00000080:00200000:1.0:1734016138.425838:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f6c0 00000080:00200000:1.0:1734016138.425838:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425839:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425841:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425844:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425848:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425851:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425855:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425856:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2707456 00000080:00200000:1.0:1734016138.425858:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 661 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425863:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2707456, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425865:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f5e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425866:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991100 2fffff00000805 3:0 ffff8800a862f5e8 661 lru 00000080:00200000:1.0:1734016138.425869:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f648 661: 1< 0 - - > 2< 2707456 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487092224 - | - - + - > 5< - - + - | 0 - | 661 - -> 00000080:00200000:1.0:1734016138.425869:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f5e8 00000080:00200000:1.0:1734016138.425870:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425870:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425873:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425876:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425879:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425882:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425886:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425887:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2711552 00000080:00200000:1.0:1734016138.425888:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 662 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425894:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2711552, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425895:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425897:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991140 2fffff00000805 3:0 ffff8800a862f510 662 lru 00000080:00200000:1.0:1734016138.425899:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f570 662: 1< 0 - - > 2< 2711552 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487088128 - | - - + - > 5< - - + - | 0 - | 662 - -> 00000080:00200000:1.0:1734016138.425900:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f510 00000080:00200000:1.0:1734016138.425900:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425901:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425903:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425906:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425909:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425912:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425916:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425918:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2715648 00000080:00200000:1.0:1734016138.425919:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 663 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425925:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2715648, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425926:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425927:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991180 2fffff00000805 3:0 ffff8800a862f438 663 lru 00000080:00200000:1.0:1734016138.425929:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f498 663: 1< 0 - - > 2< 2715648 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487084032 - | - - + - > 5< - - + - | 0 - | 663 - -> 00000080:00200000:1.0:1734016138.425930:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f438 00000080:00200000:1.0:1734016138.425931:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425931:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425934:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425937:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425940:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425944:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425948:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425949:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2719744 00000080:00200000:1.0:1734016138.425950:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 664 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.425956:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2719744, len 4096, copied 4096 00000080:00200000:1.0:1734016138.425958:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.425960:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049911c0 2fffff00000805 3:0 ffff8800a862f360 664 lru 00000080:00200000:1.0:1734016138.425963:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f3c0 664: 1< 0 - - > 2< 2719744 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487079936 - | - - + - > 5< - - + - | 0 - | 664 - -> 00000080:00200000:1.0:1734016138.425964:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f360 00000080:00200000:1.0:1734016138.425965:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.425966:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.425970:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425975:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425980:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.425986:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.425993:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.425996:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2723840 00000080:00200000:1.0:1734016138.425998:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 665 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426013:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2723840, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426016:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426017:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991200 2fffff00000805 3:0 ffff8800a862f288 665 lru 00000080:00200000:1.0:1734016138.426020:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f2e8 665: 1< 0 - - > 2< 2723840 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487075840 - | - - + - > 5< - - + - | 0 - | 665 - -> 00000080:00200000:1.0:1734016138.426022:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f288 00000080:00200000:1.0:1734016138.426022:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426023:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426029:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426035:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426039:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426042:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426049:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426053:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2727936 00000080:00200000:1.0:1734016138.426055:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 666 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426063:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2727936, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426066:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f1b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426067:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991240 2fffff00000805 3:0 ffff8800a862f1b0 666 lru 00000080:00200000:1.0:1734016138.426071:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f210 666: 1< 0 - - > 2< 2727936 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487071744 - | - - + - > 5< - - + - | 0 - | 666 - -> 00000080:00200000:1.0:1734016138.426073:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f1b0 00000080:00200000:1.0:1734016138.426074:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426075:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426081:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426086:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426091:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426095:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426102:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426105:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2732032 00000080:00200000:1.0:1734016138.426108:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 667 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426129:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2732032, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426133:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f0d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426135:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991280 2fffff00000825 2:0 ffff8800a862f0d8 667 lru 00000080:00200000:1.0:1734016138.426138:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f138 667: 1< 0 - - > 2< 2732032 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487067648 - | - - + - > 5< - - + - | 0 - | 667 - -> 00000080:00200000:1.0:1734016138.426139:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f0d8 00000080:00200000:1.0:1734016138.426140:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426141:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426145:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426152:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426157:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426162:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426169:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426172:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2736128 00000080:00200000:1.0:1734016138.426174:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 668 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426184:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2736128, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426188:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a862f000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426190:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049912c0 2fffff00000805 3:0 ffff8800a862f000 668 lru 00000080:00200000:1.0:1734016138.426193:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a862f060 668: 1< 0 - - > 2< 2736128 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487063552 - | - - + - > 5< - - + - | 0 - | 668 - -> 00000080:00200000:1.0:1734016138.426194:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a862f000 00000080:00200000:1.0:1734016138.426195:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426195:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426199:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426205:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426211:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426215:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426220:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426223:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2740224 00000080:00200000:1.0:1734016138.426225:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 669 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426235:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2740224, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426238:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426239:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991300 2fffff00000805 3:0 ffff8800a71c4e58 669 lru 00000080:00200000:1.0:1734016138.426242:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4eb8 669: 1< 0 - - > 2< 2740224 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487059456 - | - - + - > 5< - - + - | 0 - | 669 - -> 00000080:00200000:1.0:1734016138.426243:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4e58 00000080:00200000:1.0:1734016138.426244:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426245:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426249:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426254:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426258:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426263:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426269:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426272:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2744320 00000080:00200000:1.0:1734016138.426274:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 670 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426284:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2744320, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426286:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426288:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991340 2fffff00000805 3:0 ffff8800a71c4d80 670 lru 00000080:00200000:1.0:1734016138.426291:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4de0 670: 1< 0 - - > 2< 2744320 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487055360 - | - - + - > 5< - - + - | 0 - | 670 - -> 00000080:00200000:1.0:1734016138.426292:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4d80 00000080:00200000:1.0:1734016138.426293:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426294:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426300:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426306:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426310:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426315:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426321:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426324:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2748416 00000080:00200000:1.0:1734016138.426326:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 671 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426333:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2748416, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426336:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426338:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991380 2fffff00000805 3:0 ffff8800a71c4ca8 671 lru 00000080:00200000:1.0:1734016138.426341:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4d08 671: 1< 0 - - > 2< 2748416 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487051264 - | - - + - > 5< - - + - | 0 - | 671 - -> 00000080:00200000:1.0:1734016138.426342:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4ca8 00000080:00200000:1.0:1734016138.426343:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426344:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426348:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426353:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426357:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426363:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426369:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426371:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2752512 00000080:00200000:1.0:1734016138.426372:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 672 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426379:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2752512, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426381:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426383:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049913c0 2fffff00000805 3:0 ffff8800a71c4bd0 672 lru 00000080:00200000:1.0:1734016138.426386:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4c30 672: 1< 0 - - > 2< 2752512 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487047168 - | - - + - > 5< - - + - | 0 - | 672 - -> 00000080:00200000:1.0:1734016138.426387:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4bd0 00000080:00200000:1.0:1734016138.426388:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426389:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426392:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426397:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426401:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426406:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426411:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426413:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2756608 00000080:00200000:1.0:1734016138.426414:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 673 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426421:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2756608, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426423:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426426:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991400 2fffff00000805 3:0 ffff8800a71c4af8 673 lru 00000080:00200000:1.0:1734016138.426429:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4b58 673: 1< 0 - - > 2< 2756608 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487043072 - | - - + - > 5< - - + - | 0 - | 673 - -> 00000080:00200000:1.0:1734016138.426430:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4af8 00000080:00200000:1.0:1734016138.426431:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426432:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426435:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426440:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426444:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426450:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426455:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426457:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2760704 00000080:00200000:1.0:1734016138.426458:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 674 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426465:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2760704, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426467:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426469:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991440 2fffff00000805 3:0 ffff8800a71c4a20 674 lru 00000080:00200000:1.0:1734016138.426472:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4a80 674: 1< 0 - - > 2< 2760704 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487038976 - | - - + - > 5< - - + - | 0 - | 674 - -> 00000080:00200000:1.0:1734016138.426473:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4a20 00000080:00200000:1.0:1734016138.426474:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426475:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426479:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426484:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426488:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426493:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426498:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426500:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2764800 00000080:00200000:1.0:1734016138.426501:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 675 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426508:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2764800, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426510:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426512:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991480 2fffff00000805 3:0 ffff8800a71c4948 675 lru 00000080:00200000:1.0:1734016138.426515:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c49a8 675: 1< 0 - - > 2< 2764800 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487034880 - | - - + - > 5< - - + - | 0 - | 675 - -> 00000080:00200000:1.0:1734016138.426516:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4948 00000080:00200000:1.0:1734016138.426517:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426518:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426521:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426525:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426530:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426535:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426541:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426543:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2768896 00000080:00200000:1.0:1734016138.426545:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 676 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426552:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2768896, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426554:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426556:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049914c0 2fffff00000805 3:0 ffff8800a71c4870 676 lru 00000080:00200000:1.0:1734016138.426559:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c48d0 676: 1< 0 - - > 2< 2768896 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487030784 - | - - + - > 5< - - + - | 0 - | 676 - -> 00000080:00200000:1.0:1734016138.426561:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4870 00000080:00200000:1.0:1734016138.426561:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426562:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426565:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426570:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426574:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426579:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426584:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426586:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2772992 00000080:00200000:1.0:1734016138.426587:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 677 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426594:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2772992, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426597:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426598:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991500 2fffff00000805 3:0 ffff8800a71c4798 677 lru 00000080:00200000:1.0:1734016138.426602:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c47f8 677: 1< 0 - - > 2< 2772992 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487026688 - | - - + - > 5< - - + - | 0 - | 677 - -> 00000080:00200000:1.0:1734016138.426603:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4798 00000080:00200000:1.0:1734016138.426604:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426605:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426608:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426613:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426617:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426621:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426627:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426629:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2777088 00000080:00200000:1.0:1734016138.426630:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 678 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426637:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2777088, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426639:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c46c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426641:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991540 2fffff00000805 3:0 ffff8800a71c46c0 678 lru 00000080:00200000:1.0:1734016138.426644:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4720 678: 1< 0 - - > 2< 2777088 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487022592 - | - - + - > 5< - - + - | 0 - | 678 - -> 00000080:00200000:1.0:1734016138.426645:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c46c0 00000080:00200000:1.0:1734016138.426646:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426647:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426651:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426655:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426660:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426664:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426669:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426671:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2781184 00000080:00200000:1.0:1734016138.426673:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 679 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426679:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2781184, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426681:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c45e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426683:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991580 2fffff00000805 3:0 ffff8800a71c45e8 679 lru 00000080:00200000:1.0:1734016138.426686:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4648 679: 1< 0 - - > 2< 2781184 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487018496 - | - - + - > 5< - - + - | 0 - | 679 - -> 00000080:00200000:1.0:1734016138.426687:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c45e8 00000080:00200000:1.0:1734016138.426688:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426689:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426727:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426733:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426736:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426741:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426749:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426751:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2785280 00000080:00200000:1.0:1734016138.426754:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 680 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426763:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2785280, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426766:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426768:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049915c0 2fffff00000805 3:0 ffff8800a71c4510 680 lru 00000080:00200000:1.0:1734016138.426772:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4570 680: 1< 0 - - > 2< 2785280 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487014400 - | - - + - > 5< - - + - | 0 - | 680 - -> 00000080:00200000:1.0:1734016138.426773:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4510 00000080:00200000:1.0:1734016138.426774:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426775:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426780:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426786:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426792:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426797:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426804:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426806:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2789376 00000080:00200000:1.0:1734016138.426808:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 681 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426818:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2789376, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426821:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426822:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991600 2fffff00000825 2:0 ffff8800a71c4438 681 lru 00000080:00200000:1.0:1734016138.426826:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4498 681: 1< 0 - - > 2< 2789376 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487010304 - | - - + - > 5< - - + - | 0 - | 681 - -> 00000080:00200000:1.0:1734016138.426828:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4438 00000080:00200000:1.0:1734016138.426829:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426830:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426834:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426839:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426844:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426848:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426854:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426856:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2793472 00000080:00200000:1.0:1734016138.426857:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 682 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426865:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2793472, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426867:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426868:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991640 2fffff00000805 3:0 ffff8800a71c4360 682 lru 00000080:00200000:1.0:1734016138.426872:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c43c0 682: 1< 0 - - > 2< 2793472 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487006208 - | - - + - > 5< - - + - | 0 - | 682 - -> 00000080:00200000:1.0:1734016138.426873:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4360 00000080:00200000:1.0:1734016138.426874:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426875:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426878:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426883:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426888:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426893:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426900:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426903:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2797568 00000080:00200000:1.0:1734016138.426904:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 683 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426915:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2797568, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426917:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426919:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991680 2fffff00000805 3:0 ffff8800a71c4288 683 lru 00000080:00200000:1.0:1734016138.426922:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c42e8 683: 1< 0 - - > 2< 2797568 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 487002112 - | - - + - > 5< - - + - | 0 - | 683 - -> 00000080:00200000:1.0:1734016138.426923:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4288 00000080:00200000:1.0:1734016138.426924:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426925:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426929:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426934:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426938:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426942:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426950:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.426952:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2801664 00000080:00200000:1.0:1734016138.426953:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 684 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.426961:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2801664, len 4096, copied 4096 00000080:00200000:1.0:1734016138.426963:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c41b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.426965:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049916c0 2fffff00000805 3:0 ffff8800a71c41b0 684 lru 00000080:00200000:1.0:1734016138.426969:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4210 684: 1< 0 - - > 2< 2801664 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486998016 - | - - + - > 5< - - + - | 0 - | 684 - -> 00000080:00200000:1.0:1734016138.426970:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c41b0 00000080:00200000:1.0:1734016138.426970:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.426971:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.426975:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426980:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426985:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.426991:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.426998:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427000:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2805760 00000080:00200000:1.0:1734016138.427002:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 685 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427015:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2805760, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427018:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c40d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427020:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991700 2fffff00000805 3:0 ffff8800a71c40d8 685 lru 00000080:00200000:1.0:1734016138.427024:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4138 685: 1< 0 - - > 2< 2805760 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486993920 - | - - + - > 5< - - + - | 0 - | 685 - -> 00000080:00200000:1.0:1734016138.427025:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c40d8 00000080:00200000:1.0:1734016138.427026:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427027:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427033:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427039:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427044:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427050:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427057:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427060:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2809856 00000080:00200000:1.0:1734016138.427061:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 686 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427079:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2809856, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427081:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71c4000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427083:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991740 2fffff00000805 3:0 ffff8800a71c4000 686 lru 00000080:00200000:1.0:1734016138.427087:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71c4060 686: 1< 0 - - > 2< 2809856 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486989824 - | - - + - > 5< - - + - | 0 - | 686 - -> 00000080:00200000:1.0:1734016138.427088:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71c4000 00000080:00200000:1.0:1734016138.427089:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427090:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427095:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427100:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427105:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427110:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427126:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427128:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2813952 00000080:00200000:1.0:1734016138.427130:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 687 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427138:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2813952, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427141:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427143:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991780 2fffff00000805 3:0 ffff8800a8f95e58 687 lru 00000080:00200000:1.0:1734016138.427146:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95eb8 687: 1< 0 - - > 2< 2813952 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486985728 - | - - + - > 5< - - + - | 0 - | 687 - -> 00000080:00200000:1.0:1734016138.427148:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95e58 00000080:00200000:1.0:1734016138.427148:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427149:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427153:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427158:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427163:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427169:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427174:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427176:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2818048 00000080:00200000:1.0:1734016138.427178:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 688 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427185:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2818048, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427187:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427189:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049917c0 2fffff00000805 3:0 ffff8800a8f95d80 688 lru 00000080:00200000:1.0:1734016138.427192:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95de0 688: 1< 0 - - > 2< 2818048 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486981632 - | - - + - > 5< - - + - | 0 - | 688 - -> 00000080:00200000:1.0:1734016138.427194:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95d80 00000080:00200000:1.0:1734016138.427194:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427195:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427199:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427204:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427208:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427213:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427218:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427220:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2822144 00000080:00200000:1.0:1734016138.427222:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 689 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427228:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2822144, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427231:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427232:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991800 2fffff00000805 3:0 ffff8800a8f95ca8 689 lru 00000080:00200000:1.0:1734016138.427236:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95d08 689: 1< 0 - - > 2< 2822144 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486977536 - | - - + - > 5< - - + - | 0 - | 689 - -> 00000080:00200000:1.0:1734016138.427237:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95ca8 00000080:00200000:1.0:1734016138.427238:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427238:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427242:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427246:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427251:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427255:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427260:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427262:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2826240 00000080:00200000:1.0:1734016138.427264:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 690 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427270:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2826240, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427272:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427274:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991840 2fffff00000805 3:0 ffff8800a8f95bd0 690 lru 00000080:00200000:1.0:1734016138.427277:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95c30 690: 1< 0 - - > 2< 2826240 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486973440 - | - - + - > 5< - - + - | 0 - | 690 - -> 00000080:00200000:1.0:1734016138.427279:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95bd0 00000080:00200000:1.0:1734016138.427279:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427280:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427288:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427293:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427298:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427303:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427308:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427310:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2830336 00000080:00200000:1.0:1734016138.427311:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 691 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427318:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2830336, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427320:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427322:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991880 2fffff00000805 3:0 ffff8800a8f95af8 691 lru 00000080:00200000:1.0:1734016138.427326:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95b58 691: 1< 0 - - > 2< 2830336 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486969344 - | - - + - > 5< - - + - | 0 - | 691 - -> 00000080:00200000:1.0:1734016138.427327:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95af8 00000080:00200000:1.0:1734016138.427327:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427328:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427331:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427337:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427341:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427347:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427352:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427354:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2834432 00000080:00200000:1.0:1734016138.427356:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 692 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427362:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2834432, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427364:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427366:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049918c0 2fffff00000805 3:0 ffff8800a8f95a20 692 lru 00000080:00200000:1.0:1734016138.427370:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95a80 692: 1< 0 - - > 2< 2834432 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486965248 - | - - + - > 5< - - + - | 0 - | 692 - -> 00000080:00200000:1.0:1734016138.427371:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95a20 00000080:00200000:1.0:1734016138.427372:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427372:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427376:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427381:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427385:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427390:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427395:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427397:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2838528 00000080:00200000:1.0:1734016138.427399:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 693 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427406:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2838528, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427408:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427409:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991900 2fffff00000805 3:0 ffff8800a8f95948 693 lru 00000080:00200000:1.0:1734016138.427413:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f959a8 693: 1< 0 - - > 2< 2838528 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486961152 - | - - + - > 5< - - + - | 0 - | 693 - -> 00000080:00200000:1.0:1734016138.427414:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95948 00000080:00200000:1.0:1734016138.427415:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427416:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427419:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427423:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427428:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427433:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427438:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427440:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2842624 00000080:00200000:1.0:1734016138.427441:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 694 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427448:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2842624, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427450:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427452:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991940 2fffff00000805 3:0 ffff8800a8f95870 694 lru 00000080:00200000:1.0:1734016138.427456:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f958d0 694: 1< 0 - - > 2< 2842624 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486957056 - | - - + - > 5< - - + - | 0 - | 694 - -> 00000080:00200000:1.0:1734016138.427457:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95870 00000080:00200000:1.0:1734016138.427457:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427458:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427462:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427466:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427470:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427475:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427480:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427482:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2846720 00000080:00200000:1.0:1734016138.427484:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 695 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427492:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2846720, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427494:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427496:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991980 2fffff00000825 2:0 ffff8800a8f95798 695 lru 00000080:00200000:1.0:1734016138.427499:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f957f8 695: 1< 0 - - > 2< 2846720 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486952960 - | - - + - > 5< - - + - | 0 - | 695 - -> 00000080:00200000:1.0:1734016138.427500:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95798 00000080:00200000:1.0:1734016138.427501:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427502:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427505:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427510:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427514:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427520:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427525:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427527:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2850816 00000080:00200000:1.0:1734016138.427528:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 696 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427535:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2850816, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427537:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f956c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427539:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049919c0 2fffff00000805 3:0 ffff8800a8f956c0 696 lru 00000080:00200000:1.0:1734016138.427542:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95720 696: 1< 0 - - > 2< 2850816 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486948864 - | - - + - > 5< - - + - | 0 - | 696 - -> 00000080:00200000:1.0:1734016138.427543:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f956c0 00000080:00200000:1.0:1734016138.427544:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427545:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427548:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427554:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427558:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427563:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427568:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427570:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2854912 00000080:00200000:1.0:1734016138.427571:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 697 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427578:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2854912, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427580:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f955e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427582:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991a00 2fffff00000805 3:0 ffff8800a8f955e8 697 lru 00000080:00200000:1.0:1734016138.427585:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95648 697: 1< 0 - - > 2< 2854912 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486944768 - | - - + - > 5< - - + - | 0 - | 697 - -> 00000080:00200000:1.0:1734016138.427586:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f955e8 00000080:00200000:1.0:1734016138.427587:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427588:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427591:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427596:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427600:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427605:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427612:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427614:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2859008 00000080:00200000:1.0:1734016138.427616:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 698 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427627:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2859008, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427629:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427631:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991a40 2fffff00000805 3:0 ffff8800a8f95510 698 lru 00000080:00200000:1.0:1734016138.427634:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95570 698: 1< 0 - - > 2< 2859008 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486940672 - | - - + - > 5< - - + - | 0 - | 698 - -> 00000080:00200000:1.0:1734016138.427635:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95510 00000080:00200000:1.0:1734016138.427636:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427637:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427642:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427648:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427652:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427657:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427664:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427668:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2863104 00000080:00200000:1.0:1734016138.427669:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 699 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427679:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2863104, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427682:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427683:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991a80 2fffff00000805 3:0 ffff8800a8f95438 699 lru 00000080:00200000:1.0:1734016138.427687:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95498 699: 1< 0 - - > 2< 2863104 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486936576 - | - - + - > 5< - - + - | 0 - | 699 - -> 00000080:00200000:1.0:1734016138.427688:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95438 00000080:00200000:1.0:1734016138.427708:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427709:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427716:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427722:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427726:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427732:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427739:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427742:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2867200 00000080:00200000:1.0:1734016138.427744:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 700 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427755:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2867200, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427757:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427759:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991ac0 2fffff00000805 3:0 ffff8800a8f95360 700 lru 00000080:00200000:1.0:1734016138.427763:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f953c0 700: 1< 0 - - > 2< 2867200 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486932480 - | - - + - > 5< - - + - | 0 - | 700 - -> 00000080:00200000:1.0:1734016138.427765:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95360 00000080:00200000:1.0:1734016138.427765:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427766:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427771:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427777:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427783:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427788:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427795:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427798:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2871296 00000080:00200000:1.0:1734016138.427800:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 701 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427813:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2871296, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427816:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427818:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991b00 2fffff00000805 3:0 ffff8800a8f95288 701 lru 00000080:00200000:1.0:1734016138.427821:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f952e8 701: 1< 0 - - > 2< 2871296 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486928384 - | - - + - > 5< - - + - | 0 - | 701 - -> 00000080:00200000:1.0:1734016138.427822:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95288 00000080:00200000:1.0:1734016138.427823:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427824:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427831:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427836:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427840:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427844:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427850:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427853:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2875392 00000080:00200000:1.0:1734016138.427854:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 702 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427864:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2875392, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427866:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f951b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427869:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991b40 2fffff00000805 3:0 ffff8800a8f951b0 702 lru 00000080:00200000:1.0:1734016138.427872:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95210 702: 1< 0 - - > 2< 2875392 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486924288 - | - - + - > 5< - - + - | 0 - | 702 - -> 00000080:00200000:1.0:1734016138.427873:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f951b0 00000080:00200000:1.0:1734016138.427874:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427875:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427880:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427885:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427890:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427896:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427903:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427906:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2879488 00000080:00200000:1.0:1734016138.427908:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 703 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427918:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2879488, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427921:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f950d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427922:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991b80 2fffff00000805 3:0 ffff8800a8f950d8 703 lru 00000080:00200000:1.0:1734016138.427927:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95138 703: 1< 0 - - > 2< 2879488 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486920192 - | - - + - > 5< - - + - | 0 - | 703 - -> 00000080:00200000:1.0:1734016138.427928:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f950d8 00000080:00200000:1.0:1734016138.427928:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427929:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427934:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427939:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427944:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427949:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.427955:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.427957:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2883584 00000080:00200000:1.0:1734016138.427959:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 704 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.427967:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2883584, len 4096, copied 4096 00000080:00200000:1.0:1734016138.427969:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8f95000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.427971:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991bc0 2fffff00000805 3:0 ffff8800a8f95000 704 lru 00000080:00200000:1.0:1734016138.427975:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8f95060 704: 1< 0 - - > 2< 2883584 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486916096 - | - - + - > 5< - - + - | 0 - | 704 - -> 00000080:00200000:1.0:1734016138.427976:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8f95000 00000080:00200000:1.0:1734016138.427976:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.427977:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.427981:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427985:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427990:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.427994:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428000:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428002:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2887680 00000080:00200000:1.0:1734016138.428003:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 705 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428012:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2887680, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428014:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fde58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428016:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991c00 2fffff00000805 3:0 ffff8800a71fde58 705 lru 00000080:00200000:1.0:1734016138.428019:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fdeb8 705: 1< 0 - - > 2< 2887680 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486912000 - | - - + - > 5< - - + - | 0 - | 705 - -> 00000080:00200000:1.0:1734016138.428020:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fde58 00000080:00200000:1.0:1734016138.428021:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428022:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428025:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428030:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428034:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428039:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428044:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428046:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2891776 00000080:00200000:1.0:1734016138.428047:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 706 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428054:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2891776, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428056:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fdd80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428058:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991c40 2fffff00000805 3:0 ffff8800a71fdd80 706 lru 00000080:00200000:1.0:1734016138.428061:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fdde0 706: 1< 0 - - > 2< 2891776 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486907904 - | - - + - > 5< - - + - | 0 - | 706 - -> 00000080:00200000:1.0:1734016138.428063:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fdd80 00000080:00200000:1.0:1734016138.428063:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428064:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428068:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428073:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428078:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428082:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428087:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428089:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2895872 00000080:00200000:1.0:1734016138.428091:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 707 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428098:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2895872, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428100:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fdca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428101:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991c80 2fffff00000805 3:0 ffff8800a71fdca8 707 lru 00000080:00200000:1.0:1734016138.428105:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fdd08 707: 1< 0 - - > 2< 2895872 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486903808 - | - - + - > 5< - - + - | 0 - | 707 - -> 00000080:00200000:1.0:1734016138.428106:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fdca8 00000080:00200000:1.0:1734016138.428107:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428107:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428111:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428125:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428129:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428135:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428142:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428144:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2899968 00000080:00200000:1.0:1734016138.428145:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 708 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428154:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2899968, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428157:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fdbd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428159:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991cc0 2fffff00000805 3:0 ffff8800a71fdbd0 708 lru 00000080:00200000:1.0:1734016138.428162:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fdc30 708: 1< 0 - - > 2< 2899968 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486899712 - | - - + - > 5< - - + - | 0 - | 708 - -> 00000080:00200000:1.0:1734016138.428164:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fdbd0 00000080:00200000:1.0:1734016138.428164:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428165:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428169:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428173:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428178:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428182:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428188:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428190:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2904064 00000080:00200000:1.0:1734016138.428192:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 709 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428201:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2904064, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428203:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fdaf8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428205:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991d00 2fffff00000825 2:0 ffff8800a71fdaf8 709 lru 00000080:00200000:1.0:1734016138.428209:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fdb58 709: 1< 0 - - > 2< 2904064 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486895616 - | - - + - > 5< - - + - | 0 - | 709 - -> 00000080:00200000:1.0:1734016138.428210:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fdaf8 00000080:00200000:1.0:1734016138.428210:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428211:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428215:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428219:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428223:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428228:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428233:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428235:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2908160 00000080:00200000:1.0:1734016138.428236:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 710 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428245:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2908160, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428247:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fda20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428248:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991d40 2fffff00000805 3:0 ffff8800a71fda20 710 lru 00000080:00200000:1.0:1734016138.428252:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fda80 710: 1< 0 - - > 2< 2908160 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486891520 - | - - + - > 5< - - + - | 0 - | 710 - -> 00000080:00200000:1.0:1734016138.428253:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fda20 00000080:00200000:1.0:1734016138.428254:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428255:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428258:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428264:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428268:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428273:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428278:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428280:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2912256 00000080:00200000:1.0:1734016138.428281:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 711 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428288:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2912256, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428290:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428291:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991d80 2fffff00000805 3:0 ffff8800a71fd948 711 lru 00000080:00200000:1.0:1734016138.428295:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd9a8 711: 1< 0 - - > 2< 2912256 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486887424 - | - - + - > 5< - - + - | 0 - | 711 - -> 00000080:00200000:1.0:1734016138.428296:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd948 00000080:00200000:1.0:1734016138.428297:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428298:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428301:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428310:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428315:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428321:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428326:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428329:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2916352 00000080:00200000:1.0:1734016138.428330:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 712 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428337:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2916352, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428339:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428341:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991dc0 2fffff00000805 3:0 ffff8800a71fd870 712 lru 00000080:00200000:1.0:1734016138.428344:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd8d0 712: 1< 0 - - > 2< 2916352 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486883328 - | - - + - > 5< - - + - | 0 - | 712 - -> 00000080:00200000:1.0:1734016138.428345:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd870 00000080:00200000:1.0:1734016138.428346:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428347:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428350:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428355:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428360:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428364:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428370:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428371:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2920448 00000080:00200000:1.0:1734016138.428373:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 713 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428380:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2920448, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428382:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428384:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991e00 2fffff00000805 3:0 ffff8800a71fd798 713 lru 00000080:00200000:1.0:1734016138.428388:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd7f8 713: 1< 0 - - > 2< 2920448 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486879232 - | - - + - > 5< - - + - | 0 - | 713 - -> 00000080:00200000:1.0:1734016138.428389:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd798 00000080:00200000:1.0:1734016138.428390:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428391:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428394:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428398:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428403:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428407:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428412:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428414:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2924544 00000080:00200000:1.0:1734016138.428416:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 714 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428422:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2924544, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428425:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd6c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428426:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991e40 2fffff00000805 3:0 ffff8800a71fd6c0 714 lru 00000080:00200000:1.0:1734016138.428430:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd720 714: 1< 0 - - > 2< 2924544 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486875136 - | - - + - > 5< - - + - | 0 - | 714 - -> 00000080:00200000:1.0:1734016138.428432:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd6c0 00000080:00200000:1.0:1734016138.428432:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428433:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428439:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428445:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428450:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428456:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428464:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428466:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2928640 00000080:00200000:1.0:1734016138.428468:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 715 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428479:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2928640, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428481:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd5e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428483:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991e80 2fffff00000805 3:0 ffff8800a71fd5e8 715 lru 00000080:00200000:1.0:1734016138.428486:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd648 715: 1< 0 - - > 2< 2928640 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486871040 - | - - + - > 5< - - + - | 0 - | 715 - -> 00000080:00200000:1.0:1734016138.428487:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd5e8 00000080:00200000:1.0:1734016138.428488:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428489:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428494:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428499:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428503:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428509:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428514:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428516:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2932736 00000080:00200000:1.0:1734016138.428517:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 716 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428526:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2932736, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428528:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428530:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991ec0 2fffff00000805 3:0 ffff8800a71fd510 716 lru 00000080:00200000:1.0:1734016138.428534:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd570 716: 1< 0 - - > 2< 2932736 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486866944 - | - - + - > 5< - - + - | 0 - | 716 - -> 00000080:00200000:1.0:1734016138.428535:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd510 00000080:00200000:1.0:1734016138.428536:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428536:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428541:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428546:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428551:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428557:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428563:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428566:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2936832 00000080:00200000:1.0:1734016138.428569:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 717 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428584:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2936832, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428588:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428589:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991f00 2fffff00000805 3:0 ffff8800a71fd438 717 lru 00000080:00200000:1.0:1734016138.428592:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd498 717: 1< 0 - - > 2< 2936832 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486862848 - | - - + - > 5< - - + - | 0 - | 717 - -> 00000080:00200000:1.0:1734016138.428593:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd438 00000080:00200000:1.0:1734016138.428594:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428595:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428601:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428606:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428609:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428613:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428620:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428622:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2940928 00000080:00200000:1.0:1734016138.428624:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 718 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428633:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2940928, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428636:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428638:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991f40 2fffff00000805 3:0 ffff8800a71fd360 718 lru 00000080:00200000:1.0:1734016138.428642:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd3c0 718: 1< 0 - - > 2< 2940928 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486858752 - | - - + - > 5< - - + - | 0 - | 718 - -> 00000080:00200000:1.0:1734016138.428643:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd360 00000080:00200000:1.0:1734016138.428644:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428645:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428651:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428657:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428661:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428665:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428672:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428674:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2945024 00000080:00200000:1.0:1734016138.428676:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 719 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428686:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2945024, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428689:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428723:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991f80 2fffff00000805 3:0 ffff8800a71fd288 719 lru 00000080:00200000:1.0:1734016138.428726:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd2e8 719: 1< 0 - - > 2< 2945024 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486854656 - | - - + - > 5< - - + - | 0 - | 719 - -> 00000080:00200000:1.0:1734016138.428727:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd288 00000080:00200000:1.0:1734016138.428728:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428729:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428737:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428741:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428745:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428752:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428759:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428761:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2949120 00000080:00200000:1.0:1734016138.428763:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 720 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428777:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2949120, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428780:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd1b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428782:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004991fc0 2fffff00000805 3:0 ffff8800a71fd1b0 720 lru 00000080:00200000:1.0:1734016138.428785:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd210 720: 1< 0 - - > 2< 2949120 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486850560 - | - - + - > 5< - - + - | 0 - | 720 - -> 00000080:00200000:1.0:1734016138.428786:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd1b0 00000080:00200000:1.0:1734016138.428787:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428788:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428793:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428798:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428804:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428810:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428816:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428819:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2953216 00000080:00200000:1.0:1734016138.428821:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 721 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428830:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2953216, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428833:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd0d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428834:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004992000 2fffff00000805 3:0 ffff8800a71fd0d8 721 lru 00000080:00200000:1.0:1734016138.428838:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd138 721: 1< 0 - - > 2< 2953216 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486846464 - | - - + - > 5< - - + - | 0 - | 721 - -> 00000080:00200000:1.0:1734016138.428839:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd0d8 00000080:00200000:1.0:1734016138.428840:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428841:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428846:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428852:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428857:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428862:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428869:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428871:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2957312 00000080:00200000:1.0:1734016138.428873:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 722 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428883:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2957312, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428885:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71fd000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428887:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004992040 2fffff00000805 3:0 ffff8800a71fd000 722 lru 00000080:00200000:1.0:1734016138.428890:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71fd060 722: 1< 0 - - > 2< 2957312 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486842368 - | - - + - > 5< - - + - | 0 - | 722 - -> 00000080:00200000:1.0:1734016138.428892:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71fd000 00000080:00200000:1.0:1734016138.428892:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428893:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428898:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428903:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428908:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428914:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428920:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428923:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2961408 00000080:00200000:1.0:1734016138.428925:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 723 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428937:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2961408, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428940:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749ce58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.428942:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004992080 2fffff00000825 2:0 ffff8800a749ce58 723 lru 00000080:00200000:1.0:1734016138.428945:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749ceb8 723: 1< 0 - - > 2< 2961408 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486838272 - | - - + - > 5< - - + - | 0 - | 723 - -> 00000080:00200000:1.0:1734016138.428947:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749ce58 00000080:00200000:1.0:1734016138.428947:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.428948:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.428954:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428959:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428964:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.428970:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.428979:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.428982:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2965504 00000080:00200000:1.0:1734016138.428985:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 724 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.428996:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2965504, len 4096, copied 4096 00000080:00200000:1.0:1734016138.428999:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749cd80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.429001:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049920c0 2fffff00000805 3:0 ffff8800a749cd80 724 lru 00000080:00200000:1.0:1734016138.429005:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749cde0 724: 1< 0 - - > 2< 2965504 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486834176 - | - - + - > 5< - - + - | 0 - | 724 - -> 00000080:00200000:1.0:1734016138.429006:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749cd80 00000080:00200000:1.0:1734016138.429007:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.429008:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.429014:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.429020:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.429025:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.429030:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.429037:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.429041:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2969600 00000080:00200000:1.0:1734016138.429043:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 725 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.429054:0:10402:0:(rw26.c:1006:ll_write_end()) pos 2969600, len 4096, copied 4096 00000080:00200000:1.0:1734016138.429056:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749cca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.429058:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004992100 2fffff00000805 3:0 ffff8800a749cca8 725 lru 00000080:00200000:1.0:1734016138.429062:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749cd08 725: 1< 0 - - > 2< 2969600 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486830080 - | - - + - > 5< - - + - | 0 - | 725 - -> 00000080:00200000:1.0:1734016138.429063:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749cca8 00000080:00200000:1.0:1734016138.429063:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.429065:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.429070:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.429074:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.429079:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.429084:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.429091:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.429094:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 2973696 00000080:00200000:1.0:1734016138.429096:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 726 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435353:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 796 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435369:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3260416, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435372:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749cbd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435374:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002af1d00 1fffff00000805 3:0 ffff8800a749cbd0 796 lru 00000080:00200000:1.0:1734016138.435378:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749cc30 796: 1< 0 - - > 2< 3260416 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486539264 - | - - + - > 5< - - + - | 0 - | 796 - -> 00000080:00200000:1.0:1734016138.435380:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749cbd0 00000080:00200000:1.0:1734016138.435380:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435382:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435389:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435395:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435401:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435407:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435415:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435418:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3264512 00000080:00200000:1.0:1734016138.435420:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 797 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435430:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3264512, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435432:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749caf8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435434:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a52bc0 1fffff00000805 3:0 ffff8800a749caf8 797 lru 00000080:00200000:1.0:1734016138.435438:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749cb58 797: 1< 0 - - > 2< 3264512 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486535168 - | - - + - > 5< - - + - | 0 - | 797 - -> 00000080:00200000:1.0:1734016138.435439:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749caf8 00000080:00200000:1.0:1734016138.435440:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435441:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435446:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435452:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435458:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435464:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435471:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435473:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3268608 00000080:00200000:1.0:1734016138.435475:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 798 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435485:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3268608, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435488:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749ca20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435490:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5bd80 1fffff00000805 3:0 ffff8800a749ca20 798 lru 00000080:00200000:1.0:1734016138.435493:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749ca80 798: 1< 0 - - > 2< 3268608 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486531072 - | - - + - > 5< - - + - | 0 - | 798 - -> 00000080:00200000:1.0:1734016138.435494:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749ca20 00000080:00200000:1.0:1734016138.435495:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435496:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435502:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435507:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435511:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435515:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435521:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435524:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3272704 00000080:00200000:1.0:1734016138.435527:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 799 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435537:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3272704, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435539:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435541:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a57cc0 1fffff00000805 3:0 ffff8800a749c948 799 lru 00000080:00200000:1.0:1734016138.435545:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c9a8 799: 1< 0 - - > 2< 3272704 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486526976 - | - - + - > 5< - - + - | 0 - | 799 - -> 00000080:00200000:1.0:1734016138.435546:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c948 00000080:00200000:1.0:1734016138.435547:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435548:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435552:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435557:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435562:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435567:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435575:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435578:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3276800 00000080:00200000:1.0:1734016138.435580:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 800 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435589:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3276800, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435592:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435594:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a8f6c0 1fffff00000805 3:0 ffff8800a749c870 800 lru 00000080:00200000:1.0:1734016138.435598:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c8d0 800: 1< 0 - - > 2< 3276800 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486522880 - | - - + - > 5< - - + - | 0 - | 800 - -> 00000080:00200000:1.0:1734016138.435600:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c870 00000080:00200000:1.0:1734016138.435601:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435602:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435607:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435613:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435618:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435623:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435630:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435633:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3280896 00000080:00200000:1.0:1734016138.435635:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 801 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435645:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3280896, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435647:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435649:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002bc6740 1fffff00000805 3:0 ffff8800a749c798 801 lru 00000080:00200000:1.0:1734016138.435653:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c7f8 801: 1< 0 - - > 2< 3280896 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486518784 - | - - + - > 5< - - + - | 0 - | 801 - -> 00000080:00200000:1.0:1734016138.435655:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c798 00000080:00200000:1.0:1734016138.435655:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435656:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435661:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435666:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435671:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435676:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435682:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435686:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3284992 00000080:00200000:1.0:1734016138.435688:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 802 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435716:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3284992, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435718:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c6c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435720:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44440 1fffff00000805 3:0 ffff8800a749c6c0 802 lru 00000080:00200000:1.0:1734016138.435724:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c720 802: 1< 0 - - > 2< 3284992 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486514688 - | - - + - > 5< - - + - | 0 - | 802 - -> 00000080:00200000:1.0:1734016138.435725:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c6c0 00000080:00200000:1.0:1734016138.435725:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435726:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435731:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435735:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435740:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435744:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435752:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435755:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3289088 00000080:00200000:1.0:1734016138.435756:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 803 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435765:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3289088, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435768:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c5e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435770:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a8f7c0 1fffff00000805 3:0 ffff8800a749c5e8 803 lru 00000080:00200000:1.0:1734016138.435773:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c648 803: 1< 0 - - > 2< 3289088 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486510592 - | - - + - > 5< - - + - | 0 - | 803 - -> 00000080:00200000:1.0:1734016138.435774:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c5e8 00000080:00200000:1.0:1734016138.435775:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435776:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435781:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435786:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435791:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435796:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435803:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435806:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3293184 00000080:00200000:1.0:1734016138.435808:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 804 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435819:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3293184, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435821:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435823:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a51c80 1fffff00000805 3:0 ffff8800a749c510 804 lru 00000080:00200000:1.0:1734016138.435826:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c570 804: 1< 0 - - > 2< 3293184 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486506496 - | - - + - > 5< - - + - | 0 - | 804 - -> 00000080:00200000:1.0:1734016138.435827:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c510 00000080:00200000:1.0:1734016138.435828:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435828:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435833:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435839:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435843:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435847:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435853:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435856:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3297280 00000080:00200000:1.0:1734016138.435858:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 805 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435866:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3297280, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435868:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435870:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a736c0 1fffff00000805 3:0 ffff8800a749c438 805 lru 00000080:00200000:1.0:1734016138.435873:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c498 805: 1< 0 - - > 2< 3297280 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486502400 - | - - + - > 5< - - + - | 0 - | 805 - -> 00000080:00200000:1.0:1734016138.435874:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c438 00000080:00200000:1.0:1734016138.435875:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435876:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435880:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435884:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435889:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435895:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435903:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435905:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3301376 00000080:00200000:1.0:1734016138.435907:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 806 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435919:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3301376, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435921:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435923:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a454c0 1fffff00000825 2:0 ffff8800a749c360 806 lru 00000080:00200000:1.0:1734016138.435926:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c3c0 806: 1< 0 - - > 2< 3301376 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486498304 - | - - + - > 5< - - + - | 0 - | 806 - -> 00000080:00200000:1.0:1734016138.435927:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c360 00000080:00200000:1.0:1734016138.435928:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435929:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435933:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435940:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435945:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435950:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.435956:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.435959:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3305472 00000080:00200000:1.0:1734016138.435961:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 807 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.435975:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3305472, len 4096, copied 4096 00000080:00200000:1.0:1734016138.435977:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.435979:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7af00 1fffff00000805 3:0 ffff8800a749c288 807 lru 00000080:00200000:1.0:1734016138.435982:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c2e8 807: 1< 0 - - > 2< 3305472 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486494208 - | - - + - > 5< - - + - | 0 - | 807 - -> 00000080:00200000:1.0:1734016138.435983:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c288 00000080:00200000:1.0:1734016138.435984:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.435984:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.435989:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435992:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.435998:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436003:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436009:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436012:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3309568 00000080:00200000:1.0:1734016138.436014:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 808 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436022:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3309568, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436025:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c1b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436027:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5c1c0 1fffff00000805 3:0 ffff8800a749c1b0 808 lru 00000080:00200000:1.0:1734016138.436030:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c210 808: 1< 0 - - > 2< 3309568 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486490112 - | - - + - > 5< - - + - | 0 - | 808 - -> 00000080:00200000:1.0:1734016138.436031:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c1b0 00000080:00200000:1.0:1734016138.436031:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436032:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436036:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436042:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436047:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436051:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436056:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436058:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3313664 00000080:00200000:1.0:1734016138.436060:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 809 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436068:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3313664, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436071:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c0d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436072:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a58e40 1fffff00000805 3:0 ffff8800a749c0d8 809 lru 00000080:00200000:1.0:1734016138.436075:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c138 809: 1< 0 - - > 2< 3313664 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486486016 - | - - + - > 5< - - + - | 0 - | 809 - -> 00000080:00200000:1.0:1734016138.436076:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c0d8 00000080:00200000:1.0:1734016138.436077:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436078:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436082:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436086:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436090:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436095:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436101:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436104:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3317760 00000080:00200000:1.0:1734016138.436106:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 810 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436126:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3317760, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436129:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a749c000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436131:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a55d80 1fffff00000805 3:0 ffff8800a749c000 810 lru 00000080:00200000:1.0:1734016138.436134:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a749c060 810: 1< 0 - - > 2< 3317760 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486481920 - | - - + - > 5< - - + - | 0 - | 810 - -> 00000080:00200000:1.0:1734016138.436135:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a749c000 00000080:00200000:1.0:1734016138.436136:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436137:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436142:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436148:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436153:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436158:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436165:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436168:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3321856 00000080:00200000:1.0:1734016138.436170:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 811 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436181:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3321856, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436184:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436186:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002d18540 1fffff00000805 3:0 ffff8800a7208e58 811 lru 00000080:00200000:1.0:1734016138.436190:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208eb8 811: 1< 0 - - > 2< 3321856 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486477824 - | - - + - > 5< - - + - | 0 - | 811 - -> 00000080:00200000:1.0:1734016138.436191:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208e58 00000080:00200000:1.0:1734016138.436192:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436193:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436197:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436201:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436207:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436212:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436217:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436220:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3325952 00000080:00200000:1.0:1734016138.436221:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 812 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436228:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3325952, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436230:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436232:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a70e40 1fffff00000805 3:0 ffff8800a7208d80 812 lru 00000080:00200000:1.0:1734016138.436236:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208de0 812: 1< 0 - - > 2< 3325952 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486473728 - | - - + - > 5< - - + - | 0 - | 812 - -> 00000080:00200000:1.0:1734016138.436237:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208d80 00000080:00200000:1.0:1734016138.436237:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436238:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436242:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436247:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436252:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436256:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436262:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436264:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3330048 00000080:00200000:1.0:1734016138.436265:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 813 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436272:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3330048, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436274:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436276:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a78100 1fffff00000805 3:0 ffff8800a7208ca8 813 lru 00000080:00200000:1.0:1734016138.436280:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208d08 813: 1< 0 - - > 2< 3330048 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486469632 - | - - + - > 5< - - + - | 0 - | 813 - -> 00000080:00200000:1.0:1734016138.436281:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208ca8 00000080:00200000:1.0:1734016138.436281:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436282:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436286:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436291:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436295:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436300:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436305:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436307:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3334144 00000080:00200000:1.0:1734016138.436308:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 814 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436315:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3334144, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436317:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436319:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a744c0 1fffff00000805 3:0 ffff8800a7208bd0 814 lru 00000080:00200000:1.0:1734016138.436323:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208c30 814: 1< 0 - - > 2< 3334144 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486465536 - | - - + - > 5< - - + - | 0 - | 814 - -> 00000080:00200000:1.0:1734016138.436324:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208bd0 00000080:00200000:1.0:1734016138.436325:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436325:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436329:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436333:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436338:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436342:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436347:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436349:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3338240 00000080:00200000:1.0:1734016138.436351:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 815 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436357:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3338240, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436360:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436362:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a577c0 1fffff00000805 3:0 ffff8800a7208af8 815 lru 00000080:00200000:1.0:1734016138.436365:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208b58 815: 1< 0 - - > 2< 3338240 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486461440 - | - - + - > 5< - - + - | 0 - | 815 - -> 00000080:00200000:1.0:1734016138.436366:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208af8 00000080:00200000:1.0:1734016138.436367:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436368:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436371:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436376:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436380:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436385:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436390:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436392:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3342336 00000080:00200000:1.0:1734016138.436393:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 816 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436400:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3342336, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436402:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436404:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a42580 1fffff00000805 3:0 ffff8800a7208a20 816 lru 00000080:00200000:1.0:1734016138.436407:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208a80 816: 1< 0 - - > 2< 3342336 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486457344 - | - - + - > 5< - - + - | 0 - | 816 - -> 00000080:00200000:1.0:1734016138.436409:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208a20 00000080:00200000:1.0:1734016138.436409:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436410:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436414:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436419:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436423:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436428:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436433:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436435:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3346432 00000080:00200000:1.0:1734016138.436436:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 817 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436443:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3346432, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436445:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436447:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002b26100 1fffff00000805 3:0 ffff8800a7208948 817 lru 00000080:00200000:1.0:1734016138.436450:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72089a8 817: 1< 0 - - > 2< 3346432 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486453248 - | - - + - > 5< - - + - | 0 - | 817 - -> 00000080:00200000:1.0:1734016138.436452:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208948 00000080:00200000:1.0:1734016138.436452:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436453:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436456:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436461:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436466:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436471:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436476:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436478:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3350528 00000080:00200000:1.0:1734016138.436479:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 818 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436486:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3350528, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436488:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436489:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7c800 1fffff00000805 3:0 ffff8800a7208870 818 lru 00000080:00200000:1.0:1734016138.436493:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72088d0 818: 1< 0 - - > 2< 3350528 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486449152 - | - - + - > 5< - - + - | 0 - | 818 - -> 00000080:00200000:1.0:1734016138.436494:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208870 00000080:00200000:1.0:1734016138.436495:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436496:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436499:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436503:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436508:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436512:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436517:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436519:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3354624 00000080:00200000:1.0:1734016138.436521:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 819 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436528:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3354624, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436530:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436532:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a57700 1fffff00000805 3:0 ffff8800a7208798 819 lru 00000080:00200000:1.0:1734016138.436535:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72087f8 819: 1< 0 - - > 2< 3354624 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486445056 - | - - + - > 5< - - + - | 0 - | 819 - -> 00000080:00200000:1.0:1734016138.436536:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208798 00000080:00200000:1.0:1734016138.436537:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436538:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436541:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436545:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436550:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436554:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436559:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436561:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3358720 00000080:00200000:1.0:1734016138.436563:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 820 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436571:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3358720, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436573:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72086c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436575:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a52540 1fffff00000825 2:0 ffff8800a72086c0 820 lru 00000080:00200000:1.0:1734016138.436579:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208720 820: 1< 0 - - > 2< 3358720 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486440960 - | - - + - > 5< - - + - | 0 - | 820 - -> 00000080:00200000:1.0:1734016138.436580:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72086c0 00000080:00200000:1.0:1734016138.436581:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436581:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436585:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436590:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436595:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436599:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436604:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436606:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3362816 00000080:00200000:1.0:1734016138.436607:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 821 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436615:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3362816, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436617:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72085e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436618:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2d8c0 1fffff00000805 3:0 ffff8800a72085e8 821 lru 00000080:00200000:1.0:1734016138.436622:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208648 821: 1< 0 - - > 2< 3362816 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486436864 - | - - + - > 5< - - + - | 0 - | 821 - -> 00000080:00200000:1.0:1734016138.436623:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72085e8 00000080:00200000:1.0:1734016138.436624:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436625:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436628:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436633:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436638:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436643:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436649:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436651:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3366912 00000080:00200000:1.0:1734016138.436652:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 822 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436659:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3366912, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436661:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436662:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2adc0 1fffff00000805 3:0 ffff8800a7208510 822 lru 00000080:00200000:1.0:1734016138.436666:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208570 822: 1< 0 - - > 2< 3366912 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486432768 - | - - + - > 5< - - + - | 0 - | 822 - -> 00000080:00200000:1.0:1734016138.436667:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208510 00000080:00200000:1.0:1734016138.436667:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436668:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436672:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436677:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436681:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436685:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436708:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436712:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3371008 00000080:00200000:1.0:1734016138.436715:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 823 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436726:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3371008, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436728:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436730:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a55a00 1fffff00000805 3:0 ffff8800a7208438 823 lru 00000080:00200000:1.0:1734016138.436735:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208498 823: 1< 0 - - > 2< 3371008 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486428672 - | - - + - > 5< - - + - | 0 - | 823 - -> 00000080:00200000:1.0:1734016138.436736:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208438 00000080:00200000:1.0:1734016138.436737:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436738:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436743:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436749:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436755:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436760:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436767:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436770:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3375104 00000080:00200000:1.0:1734016138.436771:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 824 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436780:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3375104, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436782:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436784:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a532c0 1fffff00000805 3:0 ffff8800a7208360 824 lru 00000080:00200000:1.0:1734016138.436788:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72083c0 824: 1< 0 - - > 2< 3375104 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486424576 - | - - + - > 5< - - + - | 0 - | 824 - -> 00000080:00200000:1.0:1734016138.436789:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208360 00000080:00200000:1.0:1734016138.436790:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436791:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436796:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436802:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436808:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436813:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436819:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436822:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3379200 00000080:00200000:1.0:1734016138.436824:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 825 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436832:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3379200, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436835:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436837:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a428c0 1fffff00000805 3:0 ffff8800a7208288 825 lru 00000080:00200000:1.0:1734016138.436842:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72082e8 825: 1< 0 - - > 2< 3379200 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486420480 - | - - + - > 5< - - + - | 0 - | 825 - -> 00000080:00200000:1.0:1734016138.436843:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208288 00000080:00200000:1.0:1734016138.436843:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436844:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436849:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436855:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436860:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436865:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436872:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436875:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3383296 00000080:00200000:1.0:1734016138.436877:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 826 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436887:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3383296, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436889:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72081b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436891:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a45400 1fffff00000805 3:0 ffff8800a72081b0 826 lru 00000080:00200000:1.0:1734016138.436895:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208210 826: 1< 0 - - > 2< 3383296 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486416384 - | - - + - > 5< - - + - | 0 - | 826 - -> 00000080:00200000:1.0:1734016138.436897:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72081b0 00000080:00200000:1.0:1734016138.436897:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436898:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436903:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436909:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436915:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436920:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436927:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436929:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3387392 00000080:00200000:1.0:1734016138.436932:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 827 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436940:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3387392, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436943:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72080d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436945:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002b8dd00 1fffff00000805 3:0 ffff8800a72080d8 827 lru 00000080:00200000:1.0:1734016138.436949:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208138 827: 1< 0 - - > 2< 3387392 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486412288 - | - - + - > 5< - - + - | 0 - | 827 - -> 00000080:00200000:1.0:1734016138.436951:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72080d8 00000080:00200000:1.0:1734016138.436951:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.436952:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.436957:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436963:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436969:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.436974:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.436981:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.436984:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3391488 00000080:00200000:1.0:1734016138.436985:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 828 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.436994:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3391488, len 4096, copied 4096 00000080:00200000:1.0:1734016138.436996:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7208000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.436998:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5bc00 1fffff00000805 3:0 ffff8800a7208000 828 lru 00000080:00200000:1.0:1734016138.437003:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7208060 828: 1< 0 - - > 2< 3391488 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486408192 - | - - + - > 5< - - + - | 0 - | 828 - -> 00000080:00200000:1.0:1734016138.437004:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7208000 00000080:00200000:1.0:1734016138.437005:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437006:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437011:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437017:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437022:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437028:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437034:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437037:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3395584 00000080:00200000:1.0:1734016138.437038:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 829 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437048:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3395584, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437050:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437053:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44500 1fffff00000805 3:0 ffff8800a7221e58 829 lru 00000080:00200000:1.0:1734016138.437057:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221eb8 829: 1< 0 - - > 2< 3395584 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486404096 - | - - + - > 5< - - + - | 0 - | 829 - -> 00000080:00200000:1.0:1734016138.437058:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221e58 00000080:00200000:1.0:1734016138.437059:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437060:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437064:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437070:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437076:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437082:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437089:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437091:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3399680 00000080:00200000:1.0:1734016138.437093:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 830 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437102:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3399680, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437105:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437107:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a45580 1fffff00000805 3:0 ffff8800a7221d80 830 lru 00000080:00200000:1.0:1734016138.437111:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221de0 830: 1< 0 - - > 2< 3399680 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486400000 - | - - + - > 5< - - + - | 0 - | 830 - -> 00000080:00200000:1.0:1734016138.437112:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221d80 00000080:00200000:1.0:1734016138.437123:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437124:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437130:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437136:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437141:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437147:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437153:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437156:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3403776 00000080:00200000:1.0:1734016138.437158:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 831 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437168:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3403776, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437171:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437173:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a74400 1fffff00000805 3:0 ffff8800a7221ca8 831 lru 00000080:00200000:1.0:1734016138.437177:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221d08 831: 1< 0 - - > 2< 3403776 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486395904 - | - - + - > 5< - - + - | 0 - | 831 - -> 00000080:00200000:1.0:1734016138.437178:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221ca8 00000080:00200000:1.0:1734016138.437179:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437180:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437185:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437191:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437196:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437201:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437207:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437210:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3407872 00000080:00200000:1.0:1734016138.437213:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 832 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437223:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3407872, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437226:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437228:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a70440 1fffff00000805 3:0 ffff8800a7221bd0 832 lru 00000080:00200000:1.0:1734016138.437232:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221c30 832: 1< 0 - - > 2< 3407872 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486391808 - | - - + - > 5< - - + - | 0 - | 832 - -> 00000080:00200000:1.0:1734016138.437234:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221bd0 00000080:00200000:1.0:1734016138.437234:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437235:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437240:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437246:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437251:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437257:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437264:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437266:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3411968 00000080:00200000:1.0:1734016138.437268:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 833 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437278:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3411968, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437281:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437283:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002b78600 1fffff00000805 3:0 ffff8800a7221af8 833 lru 00000080:00200000:1.0:1734016138.437287:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221b58 833: 1< 0 - - > 2< 3411968 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486387712 - | - - + - > 5< - - + - | 0 - | 833 - -> 00000080:00200000:1.0:1734016138.437288:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221af8 00000080:00200000:1.0:1734016138.437289:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437290:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437294:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437299:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437305:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437311:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437318:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437320:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3416064 00000080:00200000:1.0:1734016138.437322:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 834 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437333:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3416064, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437335:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437337:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002d18400 1fffff00000825 2:0 ffff8800a7221a20 834 lru 00000080:00200000:1.0:1734016138.437342:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221a80 834: 1< 0 - - > 2< 3416064 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486383616 - | - - + - > 5< - - + - | 0 - | 834 - -> 00000080:00200000:1.0:1734016138.437343:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221a20 00000080:00200000:1.0:1734016138.437344:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437345:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437349:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437355:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437361:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437366:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437373:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437375:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3420160 00000080:00200000:1.0:1734016138.437377:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 835 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437386:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3420160, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437389:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437391:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002afb540 1fffff00000805 3:0 ffff8800a7221948 835 lru 00000080:00200000:1.0:1734016138.437395:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72219a8 835: 1< 0 - - > 2< 3420160 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486379520 - | - - + - > 5< - - + - | 0 - | 835 - -> 00000080:00200000:1.0:1734016138.437396:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221948 00000080:00200000:1.0:1734016138.437397:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437398:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437402:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437406:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437412:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437418:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437425:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437427:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3424256 00000080:00200000:1.0:1734016138.437429:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 836 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437437:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3424256, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437440:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437442:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a42280 1fffff00000805 3:0 ffff8800a7221870 836 lru 00000080:00200000:1.0:1734016138.437446:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72218d0 836: 1< 0 - - > 2< 3424256 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486375424 - | - - + - > 5< - - + - | 0 - | 836 - -> 00000080:00200000:1.0:1734016138.437447:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221870 00000080:00200000:1.0:1734016138.437448:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437449:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437453:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437459:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437465:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437470:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437476:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437478:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3428352 00000080:00200000:1.0:1734016138.437480:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 837 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437488:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3428352, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437490:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437492:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a45b40 1fffff00000805 3:0 ffff8800a7221798 837 lru 00000080:00200000:1.0:1734016138.437496:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72217f8 837: 1< 0 - - > 2< 3428352 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486371328 - | - - + - > 5< - - + - | 0 - | 837 - -> 00000080:00200000:1.0:1734016138.437497:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221798 00000080:00200000:1.0:1734016138.437498:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437499:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437503:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437509:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437514:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437520:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437526:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437529:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3432448 00000080:00200000:1.0:1734016138.437531:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 838 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437545:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3432448, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437548:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72216c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437550:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5d040 1fffff00000805 3:0 ffff8800a72216c0 838 lru 00000080:00200000:1.0:1734016138.437554:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221720 838: 1< 0 - - > 2< 3432448 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486367232 - | - - + - > 5< - - + - | 0 - | 838 - -> 00000080:00200000:1.0:1734016138.437555:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72216c0 00000080:00200000:1.0:1734016138.437556:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437557:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437560:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437566:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437572:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437577:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437583:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437586:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3436544 00000080:00200000:1.0:1734016138.437588:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 839 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437596:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3436544, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437598:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72215e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437600:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a50e80 1fffff00000805 3:0 ffff8800a72215e8 839 lru 00000080:00200000:1.0:1734016138.437604:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221648 839: 1< 0 - - > 2< 3436544 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486363136 - | - - + - > 5< - - + - | 0 - | 839 - -> 00000080:00200000:1.0:1734016138.437606:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72215e8 00000080:00200000:1.0:1734016138.437606:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437608:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437612:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437618:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437622:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437627:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437633:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437636:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3440640 00000080:00200000:1.0:1734016138.437638:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 840 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437646:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3440640, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437648:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437650:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2e640 1fffff00000805 3:0 ffff8800a7221510 840 lru 00000080:00200000:1.0:1734016138.437655:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221570 840: 1< 0 - - > 2< 3440640 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486359040 - | - - + - > 5< - - + - | 0 - | 840 - -> 00000080:00200000:1.0:1734016138.437656:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221510 00000080:00200000:1.0:1734016138.437657:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437657:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437662:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437668:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437672:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437678:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437685:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437687:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3444736 00000080:00200000:1.0:1734016138.437689:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 841 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437713:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3444736, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437716:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437717:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a89440 1fffff00000805 3:0 ffff8800a7221438 841 lru 00000080:00200000:1.0:1734016138.437721:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221498 841: 1< 0 - - > 2< 3444736 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486354944 - | - - + - > 5< - - + - | 0 - | 841 - -> 00000080:00200000:1.0:1734016138.437722:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221438 00000080:00200000:1.0:1734016138.437723:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437724:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437729:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437735:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437740:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437745:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437752:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437755:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3448832 00000080:00200000:1.0:1734016138.437757:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 842 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437766:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3448832, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437770:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437771:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a71fc0 1fffff00000805 3:0 ffff8800a7221360 842 lru 00000080:00200000:1.0:1734016138.437775:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72213c0 842: 1< 0 - - > 2< 3448832 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486350848 - | - - + - > 5< - - + - | 0 - | 842 - -> 00000080:00200000:1.0:1734016138.437776:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221360 00000080:00200000:1.0:1734016138.437777:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437777:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437782:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437788:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437793:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437798:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437804:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437806:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3452928 00000080:00200000:1.0:1734016138.437808:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 843 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437816:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3452928, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437818:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437820:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a54a40 1fffff00000805 3:0 ffff8800a7221288 843 lru 00000080:00200000:1.0:1734016138.437824:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a72212e8 843: 1< 0 - - > 2< 3452928 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486346752 - | - - + - > 5< - - + - | 0 - | 843 - -> 00000080:00200000:1.0:1734016138.437825:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221288 00000080:00200000:1.0:1734016138.437826:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437827:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437831:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437835:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437840:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437846:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437851:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437853:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3457024 00000080:00200000:1.0:1734016138.437855:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 844 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437862:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3457024, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437864:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72211b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437866:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a73540 1fffff00000805 3:0 ffff8800a72211b0 844 lru 00000080:00200000:1.0:1734016138.437869:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221210 844: 1< 0 - - > 2< 3457024 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486342656 - | - - + - > 5< - - + - | 0 - | 844 - -> 00000080:00200000:1.0:1734016138.437870:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72211b0 00000080:00200000:1.0:1734016138.437871:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437872:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437876:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437881:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437886:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437891:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437896:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437898:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3461120 00000080:00200000:1.0:1734016138.437899:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 845 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437906:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3461120, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437908:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a72210d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437910:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a71a80 1fffff00000805 3:0 ffff8800a72210d8 845 lru 00000080:00200000:1.0:1734016138.437913:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221138 845: 1< 0 - - > 2< 3461120 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486338560 - | - - + - > 5< - - + - | 0 - | 845 - -> 00000080:00200000:1.0:1734016138.437915:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a72210d8 00000080:00200000:1.0:1734016138.437915:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437916:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437919:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437924:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437929:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437934:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437940:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437942:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3465216 00000080:00200000:1.0:1734016138.437943:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 846 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437950:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3465216, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437952:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7221000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437954:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2d3c0 1fffff00000805 3:0 ffff8800a7221000 846 lru 00000080:00200000:1.0:1734016138.437957:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7221060 846: 1< 0 - - > 2< 3465216 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486334464 - | - - + - > 5< - - + - | 0 - | 846 - -> 00000080:00200000:1.0:1734016138.437958:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7221000 00000080:00200000:1.0:1734016138.437959:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.437960:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.437963:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437968:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437972:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.437976:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.437982:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.437985:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3469312 00000080:00200000:1.0:1734016138.437987:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 847 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.437994:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3469312, len 4096, copied 4096 00000080:00200000:1.0:1734016138.437996:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.437998:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2d100 1fffff00000805 3:0 ffff8800a85c4e58 847 lru 00000080:00200000:1.0:1734016138.438002:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4eb8 847: 1< 0 - - > 2< 3469312 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486330368 - | - - + - > 5< - - + - | 0 - | 847 - -> 00000080:00200000:1.0:1734016138.438003:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4e58 00000080:00200000:1.0:1734016138.438004:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438004:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438008:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438012:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438017:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438022:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438027:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438029:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3473408 00000080:00200000:1.0:1734016138.438031:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 848 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438039:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3473408, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438041:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438043:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7c980 1fffff00000825 2:0 ffff8800a85c4d80 848 lru 00000080:00200000:1.0:1734016138.438046:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4de0 848: 1< 0 - - > 2< 3473408 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486326272 - | - - + - > 5< - - + - | 0 - | 848 - -> 00000080:00200000:1.0:1734016138.438048:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4d80 00000080:00200000:1.0:1734016138.438048:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438049:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438053:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438058:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438062:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438067:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438074:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438076:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3477504 00000080:00200000:1.0:1734016138.438077:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 849 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438089:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3477504, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438090:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438093:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5ab40 1fffff00000805 3:0 ffff8800a85c4ca8 849 lru 00000080:00200000:1.0:1734016138.438097:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4d08 849: 1< 0 - - > 2< 3477504 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486322176 - | - - + - > 5< - - + - | 0 - | 849 - -> 00000080:00200000:1.0:1734016138.438098:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4ca8 00000080:00200000:1.0:1734016138.438099:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438100:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438104:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438109:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438123:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438130:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438137:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438140:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3481600 00000080:00200000:1.0:1734016138.438142:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 850 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438152:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3481600, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438155:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438157:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a58900 1fffff00000805 3:0 ffff8800a85c4bd0 850 lru 00000080:00200000:1.0:1734016138.438160:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4c30 850: 1< 0 - - > 2< 3481600 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486318080 - | - - + - > 5< - - + - | 0 - | 850 - -> 00000080:00200000:1.0:1734016138.438161:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4bd0 00000080:00200000:1.0:1734016138.438162:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438164:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438168:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438174:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438178:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438183:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438189:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438192:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3485696 00000080:00200000:1.0:1734016138.438194:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 851 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438203:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3485696, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438206:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438207:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a56100 1fffff00000805 3:0 ffff8800a85c4af8 851 lru 00000080:00200000:1.0:1734016138.438211:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4b58 851: 1< 0 - - > 2< 3485696 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486313984 - | - - + - > 5< - - + - | 0 - | 851 - -> 00000080:00200000:1.0:1734016138.438212:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4af8 00000080:00200000:1.0:1734016138.438213:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438214:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438219:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438224:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438229:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438233:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438239:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438242:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3489792 00000080:00200000:1.0:1734016138.438243:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 852 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438253:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3489792, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438256:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438258:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5ecc0 1fffff00000805 3:0 ffff8800a85c4a20 852 lru 00000080:00200000:1.0:1734016138.438262:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4a80 852: 1< 0 - - > 2< 3489792 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486309888 - | - - + - > 5< - - + - | 0 - | 852 - -> 00000080:00200000:1.0:1734016138.438263:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4a20 00000080:00200000:1.0:1734016138.438264:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438265:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438270:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438275:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438281:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438286:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438292:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438295:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3493888 00000080:00200000:1.0:1734016138.438297:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 853 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438305:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3493888, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438307:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438309:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2d180 1fffff00000805 3:0 ffff8800a85c4948 853 lru 00000080:00200000:1.0:1734016138.438313:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c49a8 853: 1< 0 - - > 2< 3493888 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486305792 - | - - + - > 5< - - + - | 0 - | 853 - -> 00000080:00200000:1.0:1734016138.438314:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4948 00000080:00200000:1.0:1734016138.438315:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438316:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438320:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438325:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438330:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438336:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438342:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438344:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3497984 00000080:00200000:1.0:1734016138.438345:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 854 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438352:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3497984, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438354:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438356:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2d200 1fffff00000805 3:0 ffff8800a85c4870 854 lru 00000080:00200000:1.0:1734016138.438360:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c48d0 854: 1< 0 - - > 2< 3497984 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486301696 - | - - + - > 5< - - + - | 0 - | 854 - -> 00000080:00200000:1.0:1734016138.438361:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4870 00000080:00200000:1.0:1734016138.438361:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438362:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438366:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438371:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438375:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438380:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438385:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438387:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3502080 00000080:00200000:1.0:1734016138.438389:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 855 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438396:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3502080, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438398:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438400:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a56480 1fffff00000805 3:0 ffff8800a85c4798 855 lru 00000080:00200000:1.0:1734016138.438404:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c47f8 855: 1< 0 - - > 2< 3502080 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486297600 - | - - + - > 5< - - + - | 0 - | 855 - -> 00000080:00200000:1.0:1734016138.438405:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4798 00000080:00200000:1.0:1734016138.438405:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438406:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438409:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438414:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438419:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438424:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438429:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438432:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3506176 00000080:00200000:1.0:1734016138.438434:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 856 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438443:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3506176, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438445:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c46c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438447:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a55640 1fffff00000805 3:0 ffff8800a85c46c0 856 lru 00000080:00200000:1.0:1734016138.438451:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4720 856: 1< 0 - - > 2< 3506176 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486293504 - | - - + - > 5< - - + - | 0 - | 856 - -> 00000080:00200000:1.0:1734016138.438452:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c46c0 00000080:00200000:1.0:1734016138.438453:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438453:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438459:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438464:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438470:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438475:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438482:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438485:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3510272 00000080:00200000:1.0:1734016138.438487:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 857 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438498:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3510272, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438501:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c45e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438503:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a51dc0 1fffff00000805 3:0 ffff8800a85c45e8 857 lru 00000080:00200000:1.0:1734016138.438506:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4648 857: 1< 0 - - > 2< 3510272 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486289408 - | - - + - > 5< - - + - | 0 - | 857 - -> 00000080:00200000:1.0:1734016138.438507:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c45e8 00000080:00200000:1.0:1734016138.438508:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438509:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438513:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438518:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438523:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438529:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438536:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438538:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3514368 00000080:00200000:1.0:1734016138.438540:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 858 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438548:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3514368, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438550:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438552:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a55bc0 1fffff00000805 3:0 ffff8800a85c4510 858 lru 00000080:00200000:1.0:1734016138.438556:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4570 858: 1< 0 - - > 2< 3514368 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486285312 - | - - + - > 5< - - + - | 0 - | 858 - -> 00000080:00200000:1.0:1734016138.438557:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4510 00000080:00200000:1.0:1734016138.438558:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438559:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438563:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438568:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438572:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438577:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438582:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438585:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3518464 00000080:00200000:1.0:1734016138.438587:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 859 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438594:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3518464, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438596:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438598:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7c680 1fffff00000805 3:0 ffff8800a85c4438 859 lru 00000080:00200000:1.0:1734016138.438601:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4498 859: 1< 0 - - > 2< 3518464 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486281216 - | - - + - > 5< - - + - | 0 - | 859 - -> 00000080:00200000:1.0:1734016138.438603:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4438 00000080:00200000:1.0:1734016138.438603:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438604:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438607:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438612:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438616:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438621:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438626:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438628:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3522560 00000080:00200000:1.0:1734016138.438629:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 860 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438637:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3522560, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438639:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438640:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002b27240 1fffff00000805 3:0 ffff8800a85c4360 860 lru 00000080:00200000:1.0:1734016138.438644:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c43c0 860: 1< 0 - - > 2< 3522560 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486277120 - | - - + - > 5< - - + - | 0 - | 860 - -> 00000080:00200000:1.0:1734016138.438645:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4360 00000080:00200000:1.0:1734016138.438646:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438646:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438650:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438655:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438659:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438664:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438669:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438671:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3526656 00000080:00200000:1.0:1734016138.438672:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 861 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438679:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3526656, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438681:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438683:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a543c0 1fffff00000805 3:0 ffff8800a85c4288 861 lru 00000080:00200000:1.0:1734016138.438686:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c42e8 861: 1< 0 - - > 2< 3526656 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486273024 - | - - + - > 5< - - + - | 0 - | 861 - -> 00000080:00200000:1.0:1734016138.438687:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4288 00000080:00200000:1.0:1734016138.438688:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438689:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438706:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438711:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438715:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438722:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438728:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438730:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3530752 00000080:00200000:1.0:1734016138.438731:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 862 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438740:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3530752, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438742:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c41b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438744:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a70980 1fffff00000825 2:0 ffff8800a85c41b0 862 lru 00000080:00200000:1.0:1734016138.438748:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4210 862: 1< 0 - - > 2< 3530752 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486268928 - | - - + - > 5< - - + - | 0 - | 862 - -> 00000080:00200000:1.0:1734016138.438749:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c41b0 00000080:00200000:1.0:1734016138.438750:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438751:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438754:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438759:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438763:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438768:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438773:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438775:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3534848 00000080:00200000:1.0:1734016138.438776:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 863 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438783:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3534848, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438785:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c40d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438787:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a78b80 1fffff00000805 3:0 ffff8800a85c40d8 863 lru 00000080:00200000:1.0:1734016138.438791:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4138 863: 1< 0 - - > 2< 3534848 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486264832 - | - - + - > 5< - - + - | 0 - | 863 - -> 00000080:00200000:1.0:1734016138.438792:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c40d8 00000080:00200000:1.0:1734016138.438793:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438793:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438797:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438801:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438805:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438810:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438815:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438817:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3538944 00000080:00200000:1.0:1734016138.438819:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 864 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438825:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3538944, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438827:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a85c4000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438829:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7b840 1fffff00000805 3:0 ffff8800a85c4000 864 lru 00000080:00200000:1.0:1734016138.438833:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a85c4060 864: 1< 0 - - > 2< 3538944 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486260736 - | - - + - > 5< - - + - | 0 - | 864 - -> 00000080:00200000:1.0:1734016138.438834:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a85c4000 00000080:00200000:1.0:1734016138.438834:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438835:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438839:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438844:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438848:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438853:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438859:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438862:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3543040 00000080:00200000:1.0:1734016138.438863:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 865 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438874:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3543040, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438877:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770de58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438878:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a423c0 1fffff00000805 3:0 ffff8800a770de58 865 lru 00000080:00200000:1.0:1734016138.438882:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770deb8 865: 1< 0 - - > 2< 3543040 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486256640 - | - - + - > 5< - - + - | 0 - | 865 - -> 00000080:00200000:1.0:1734016138.438884:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770de58 00000080:00200000:1.0:1734016138.438884:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438885:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438891:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438897:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438902:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438908:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438915:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438917:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3547136 00000080:00200000:1.0:1734016138.438919:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 866 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438927:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3547136, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438929:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770dd80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438931:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00029e8400 1fffff00000805 3:0 ffff8800a770dd80 866 lru 00000080:00200000:1.0:1734016138.438935:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770dde0 866: 1< 0 - - > 2< 3547136 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486252544 - | - - + - > 5< - - + - | 0 - | 866 - -> 00000080:00200000:1.0:1734016138.438937:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770dd80 00000080:00200000:1.0:1734016138.438937:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438938:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438942:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438947:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438952:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438956:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.438961:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.438963:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3551232 00000080:00200000:1.0:1734016138.438965:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 867 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.438972:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3551232, len 4096, copied 4096 00000080:00200000:1.0:1734016138.438974:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770dca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.438976:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a2d340 1fffff00000805 3:0 ffff8800a770dca8 867 lru 00000080:00200000:1.0:1734016138.438980:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770dd08 867: 1< 0 - - > 2< 3551232 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486248448 - | - - + - > 5< - - + - | 0 - | 867 - -> 00000080:00200000:1.0:1734016138.438981:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770dca8 00000080:00200000:1.0:1734016138.438982:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.438983:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.438986:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438990:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438995:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.438999:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439004:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439006:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3555328 00000080:00200000:1.0:1734016138.439008:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 868 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439014:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3555328, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439016:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770dbd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439018:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5d780 1fffff00000805 3:0 ffff8800a770dbd0 868 lru 00000080:00200000:1.0:1734016138.439022:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770dc30 868: 1< 0 - - > 2< 3555328 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486244352 - | - - + - > 5< - - + - | 0 - | 868 - -> 00000080:00200000:1.0:1734016138.439023:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770dbd0 00000080:00200000:1.0:1734016138.439024:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439024:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439028:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439033:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439038:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439042:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439047:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439049:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3559424 00000080:00200000:1.0:1734016138.439051:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 869 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439065:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3559424, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439067:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770daf8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439068:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002afab80 1fffff00000805 3:0 ffff8800a770daf8 869 lru 00000080:00200000:1.0:1734016138.439072:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770db58 869: 1< 0 - - > 2< 3559424 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486240256 - | - - + - > 5< - - + - | 0 - | 869 - -> 00000080:00200000:1.0:1734016138.439073:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770daf8 00000080:00200000:1.0:1734016138.439074:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439074:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439078:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439082:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439087:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439093:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439098:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439100:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3563520 00000080:00200000:1.0:1734016138.439101:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 870 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439108:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3563520, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439110:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770da20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439112:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a747c0 1fffff00000805 3:0 ffff8800a770da20 870 lru 00000080:00200000:1.0:1734016138.439276:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770da80 870: 1< 0 - - > 2< 3563520 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486236160 - | - - + - > 5< - - + - | 0 - | 870 - -> 00000080:00200000:1.0:1734016138.439278:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770da20 00000080:00200000:1.0:1734016138.439278:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439279:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439286:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439290:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439295:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439299:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439306:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439308:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3567616 00000080:00200000:1.0:1734016138.439311:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 871 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439320:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3567616, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439322:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439324:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7ddc0 1fffff00000805 3:0 ffff8800a770d948 871 lru 00000080:00200000:1.0:1734016138.439328:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d9a8 871: 1< 0 - - > 2< 3567616 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486232064 - | - - + - > 5< - - + - | 0 - | 871 - -> 00000080:00200000:1.0:1734016138.439329:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d948 00000080:00200000:1.0:1734016138.439330:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439330:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439334:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439338:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439343:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439347:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439353:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439355:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3571712 00000080:00200000:1.0:1734016138.439356:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 872 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439363:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3571712, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439365:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439366:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5c480 1fffff00000805 3:0 ffff8800a770d870 872 lru 00000080:00200000:1.0:1734016138.439370:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d8d0 872: 1< 0 - - > 2< 3571712 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486227968 - | - - + - > 5< - - + - | 0 - | 872 - -> 00000080:00200000:1.0:1734016138.439371:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d870 00000080:00200000:1.0:1734016138.439372:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439373:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439376:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439381:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439386:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439391:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439397:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439399:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3575808 00000080:00200000:1.0:1734016138.439401:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 873 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439410:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3575808, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439412:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439414:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a51900 1fffff00000805 3:0 ffff8800a770d798 873 lru 00000080:00200000:1.0:1734016138.439418:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d7f8 873: 1< 0 - - > 2< 3575808 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486223872 - | - - + - > 5< - - + - | 0 - | 873 - -> 00000080:00200000:1.0:1734016138.439419:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d798 00000080:00200000:1.0:1734016138.439420:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439421:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439426:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439432:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439437:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439443:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439450:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439453:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3579904 00000080:00200000:1.0:1734016138.439456:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 874 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439468:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3579904, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439471:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d6c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439472:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a52e80 1fffff00000805 3:0 ffff8800a770d6c0 874 lru 00000080:00200000:1.0:1734016138.439476:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d720 874: 1< 0 - - > 2< 3579904 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486219776 - | - - + - > 5< - - + - | 0 - | 874 - -> 00000080:00200000:1.0:1734016138.439477:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d6c0 00000080:00200000:1.0:1734016138.439477:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439478:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439483:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439488:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439493:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439498:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439505:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439508:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3584000 00000080:00200000:1.0:1734016138.439510:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 875 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439519:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3584000, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439522:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d5e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439524:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a73a80 1fffff00000805 3:0 ffff8800a770d5e8 875 lru 00000080:00200000:1.0:1734016138.439528:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d648 875: 1< 0 - - > 2< 3584000 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486215680 - | - - + - > 5< - - + - | 0 - | 875 - -> 00000080:00200000:1.0:1734016138.439529:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d5e8 00000080:00200000:1.0:1734016138.439530:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439531:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439535:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439540:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439545:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439549:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439557:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439560:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3588096 00000080:00200000:1.0:1734016138.439563:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 876 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439580:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3588096, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439583:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439585:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002af6d40 1fffff00000825 2:0 ffff8800a770d510 876 lru 00000080:00200000:1.0:1734016138.439588:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d570 876: 1< 0 - - > 2< 3588096 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486211584 - | - - + - > 5< - - + - | 0 - | 876 - -> 00000080:00200000:1.0:1734016138.439589:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d510 00000080:00200000:1.0:1734016138.439590:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439591:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439595:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439602:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439606:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439610:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439618:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439620:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3592192 00000080:00200000:1.0:1734016138.439622:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 877 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439635:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3592192, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439637:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439638:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a54040 1fffff00000805 3:0 ffff8800a770d438 877 lru 00000080:00200000:1.0:1734016138.439641:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d498 877: 1< 0 - - > 2< 3592192 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486207488 - | - - + - > 5< - - + - | 0 - | 877 - -> 00000080:00200000:1.0:1734016138.439642:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d438 00000080:00200000:1.0:1734016138.439643:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439644:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439648:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439652:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439656:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439663:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439669:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439671:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3596288 00000080:00200000:1.0:1734016138.439673:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 878 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439680:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3596288, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439682:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439684:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002afcc40 1fffff00000805 3:0 ffff8800a770d360 878 lru 00000080:00200000:1.0:1734016138.439687:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d3c0 878: 1< 0 - - > 2< 3596288 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486203392 - | - - + - > 5< - - + - | 0 - | 878 - -> 00000080:00200000:1.0:1734016138.439688:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d360 00000080:00200000:1.0:1734016138.439689:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439782:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439791:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439797:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439801:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439806:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439814:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439817:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3600384 00000080:00200000:1.0:1734016138.439820:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 879 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439832:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3600384, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439838:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439839:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44380 1fffff00000805 3:0 ffff8800a770d288 879 lru 00000080:00200000:1.0:1734016138.439843:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d2e8 879: 1< 0 - - > 2< 3600384 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486199296 - | - - + - > 5< - - + - | 0 - | 879 - -> 00000080:00200000:1.0:1734016138.439843:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d288 00000080:00200000:1.0:1734016138.439844:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439845:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439850:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439855:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439859:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439864:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439870:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439871:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3604480 00000080:00200000:1.0:1734016138.439873:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 880 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439882:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3604480, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439885:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d1b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439887:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002b24380 1fffff00000805 3:0 ffff8800a770d1b0 880 lru 00000080:00200000:1.0:1734016138.439890:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d210 880: 1< 0 - - > 2< 3604480 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486195200 - | - - + - > 5< - - + - | 0 - | 880 - -> 00000080:00200000:1.0:1734016138.439892:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d1b0 00000080:00200000:1.0:1734016138.439892:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439893:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439898:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439903:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439907:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439912:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439918:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439920:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3608576 00000080:00200000:1.0:1734016138.439923:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 881 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439932:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3608576, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439934:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d0d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439936:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a8f300 1fffff00000805 3:0 ffff8800a770d0d8 881 lru 00000080:00200000:1.0:1734016138.439940:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d138 881: 1< 0 - - > 2< 3608576 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486191104 - | - - + - > 5< - - + - | 0 - | 881 - -> 00000080:00200000:1.0:1734016138.439941:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d0d8 00000080:00200000:1.0:1734016138.439942:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439943:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.439949:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439954:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439960:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.439966:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.439973:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.439976:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3612672 00000080:00200000:1.0:1734016138.439978:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 882 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.439987:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3612672, len 4096, copied 4096 00000080:00200000:1.0:1734016138.439989:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a770d000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.439991:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002af41c0 1fffff00000805 3:0 ffff8800a770d000 882 lru 00000080:00200000:1.0:1734016138.439995:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a770d060 882: 1< 0 - - > 2< 3612672 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486187008 - | - - + - > 5< - - + - | 0 - | 882 - -> 00000080:00200000:1.0:1734016138.439997:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a770d000 00000080:00200000:1.0:1734016138.439997:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.439998:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440002:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440007:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440012:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440017:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440022:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440024:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3616768 00000080:00200000:1.0:1734016138.440026:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 883 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440039:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3616768, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440041:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440043:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a589c0 1fffff00000805 3:0 ffff8800a8633000 883 lru 00000080:00200000:1.0:1734016138.440047:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633060 883: 1< 0 - - > 2< 3616768 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486182912 - | - - + - > 5< - - + - | 0 - | 883 - -> 00000080:00200000:1.0:1734016138.440048:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633000 00000080:00200000:1.0:1734016138.440049:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440049:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440053:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440058:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440062:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440068:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440074:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440076:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3620864 00000080:00200000:1.0:1734016138.440077:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 884 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440085:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3620864, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440087:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a86330d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440089:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44180 1fffff00000805 3:0 ffff8800a86330d8 884 lru 00000080:00200000:1.0:1734016138.440092:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633138 884: 1< 0 - - > 2< 3620864 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486178816 - | - - + - > 5< - - + - | 0 - | 884 - -> 00000080:00200000:1.0:1734016138.440094:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a86330d8 00000080:00200000:1.0:1734016138.440094:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440095:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440099:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440104:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440109:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440122:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440128:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440130:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3624960 00000080:00200000:1.0:1734016138.440132:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 885 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440140:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3624960, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440142:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a86331b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440144:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a55400 1fffff00000805 3:0 ffff8800a86331b0 885 lru 00000080:00200000:1.0:1734016138.440148:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633210 885: 1< 0 - - > 2< 3624960 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486174720 - | - - + - > 5< - - + - | 0 - | 885 - -> 00000080:00200000:1.0:1734016138.440149:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a86331b0 00000080:00200000:1.0:1734016138.440150:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440151:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440154:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440159:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440164:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440170:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440175:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440178:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3629056 00000080:00200000:1.0:1734016138.440179:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 886 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440186:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3629056, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440188:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440189:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002af5680 1fffff00000805 3:0 ffff8800a8633288 886 lru 00000080:00200000:1.0:1734016138.440193:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a86332e8 886: 1< 0 - - > 2< 3629056 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486170624 - | - - + - > 5< - - + - | 0 - | 886 - -> 00000080:00200000:1.0:1734016138.440194:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633288 00000080:00200000:1.0:1734016138.440195:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440196:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440199:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440204:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440209:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440213:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440219:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440221:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3633152 00000080:00200000:1.0:1734016138.440223:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 887 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440233:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3633152, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440235:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440236:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7df80 1fffff00000805 3:0 ffff8800a8633360 887 lru 00000080:00200000:1.0:1734016138.440239:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a86333c0 887: 1< 0 - - > 2< 3633152 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486166528 - | - - + - > 5< - - + - | 0 - | 887 - -> 00000080:00200000:1.0:1734016138.440240:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633360 00000080:00200000:1.0:1734016138.440241:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440242:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440246:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440249:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440254:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440258:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440264:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440266:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3637248 00000080:00200000:1.0:1734016138.440267:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 888 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440276:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3637248, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440279:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440280:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a73a40 1fffff00000805 3:0 ffff8800a8633438 888 lru 00000080:00200000:1.0:1734016138.440284:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633498 888: 1< 0 - - > 2< 3637248 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486162432 - | - - + - > 5< - - + - | 0 - | 888 - -> 00000080:00200000:1.0:1734016138.440285:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633438 00000080:00200000:1.0:1734016138.440286:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440287:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440292:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440297:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440302:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440307:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440313:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440315:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3641344 00000080:00200000:1.0:1734016138.440317:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 889 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440327:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3641344, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440329:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440331:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00029a5680 1fffff00000805 3:0 ffff8800a8633510 889 lru 00000080:00200000:1.0:1734016138.440334:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633570 889: 1< 0 - - > 2< 3641344 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486158336 - | - - + - > 5< - - + - | 0 - | 889 - -> 00000080:00200000:1.0:1734016138.440335:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633510 00000080:00200000:1.0:1734016138.440336:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440337:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440341:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440347:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440352:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440357:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440364:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440367:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3645440 00000080:00200000:1.0:1734016138.440369:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 890 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440381:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3645440, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440384:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a86335e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440385:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5c100 1fffff00000825 2:0 ffff8800a86335e8 890 lru 00000080:00200000:1.0:1734016138.440389:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633648 890: 1< 0 - - > 2< 3645440 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486154240 - | - - + - > 5< - - + - | 0 - | 890 - -> 00000080:00200000:1.0:1734016138.440391:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a86335e8 00000080:00200000:1.0:1734016138.440392:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440393:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440398:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440404:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440409:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440415:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440421:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440424:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3649536 00000080:00200000:1.0:1734016138.440427:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 891 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440446:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3649536, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440450:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a86336c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440452:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a45900 1fffff00000805 3:0 ffff8800a86336c0 891 lru 00000080:00200000:1.0:1734016138.440456:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633720 891: 1< 0 - - > 2< 3649536 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486150144 - | - - + - > 5< - - + - | 0 - | 891 - -> 00000080:00200000:1.0:1734016138.440457:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a86336c0 00000080:00200000:1.0:1734016138.440457:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440459:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440463:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440469:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440474:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440480:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440488:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440491:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3653632 00000080:00200000:1.0:1734016138.440494:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 892 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440506:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3653632, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440510:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440511:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002bbb780 1fffff00000805 3:0 ffff8800a8633798 892 lru 00000080:00200000:1.0:1734016138.440514:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a86337f8 892: 1< 0 - - > 2< 3653632 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486146048 - | - - + - > 5< - - + - | 0 - | 892 - -> 00000080:00200000:1.0:1734016138.440515:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633798 00000080:00200000:1.0:1734016138.440516:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440517:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440522:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440527:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440532:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440537:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440544:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440547:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3657728 00000080:00200000:1.0:1734016138.440549:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 893 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440559:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3657728, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440563:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440564:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a73580 1fffff00000805 3:0 ffff8800a8633870 893 lru 00000080:00200000:1.0:1734016138.440568:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a86338d0 893: 1< 0 - - > 2< 3657728 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486141952 - | - - + - > 5< - - + - | 0 - | 893 - -> 00000080:00200000:1.0:1734016138.440569:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633870 00000080:00200000:1.0:1734016138.440570:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440571:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440576:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440582:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440587:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440591:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440598:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440601:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3661824 00000080:00200000:1.0:1734016138.440603:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 894 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440613:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3661824, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440616:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440618:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a75d80 1fffff00000805 3:0 ffff8800a8633948 894 lru 00000080:00200000:1.0:1734016138.440622:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a86339a8 894: 1< 0 - - > 2< 3661824 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486137856 - | - - + - > 5< - - + - | 0 - | 894 - -> 00000080:00200000:1.0:1734016138.440623:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633948 00000080:00200000:1.0:1734016138.440624:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440625:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440629:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440635:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440639:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440644:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440650:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440652:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3665920 00000080:00200000:1.0:1734016138.440654:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 895 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440661:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3665920, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440664:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440665:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00029a5200 1fffff00000805 3:0 ffff8800a8633a20 895 lru 00000080:00200000:1.0:1734016138.440669:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633a80 895: 1< 0 - - > 2< 3665920 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486133760 - | - - + - > 5< - - + - | 0 - | 895 - -> 00000080:00200000:1.0:1734016138.440670:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633a20 00000080:00200000:1.0:1734016138.440671:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440672:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440675:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440680:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440684:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440756:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440764:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440768:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3670016 00000080:00200000:1.0:1734016138.440770:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 896 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440781:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3670016, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440784:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440785:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a56b00 1fffff00000805 3:0 ffff8800a8633af8 896 lru 00000080:00200000:1.0:1734016138.440789:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633b58 896: 1< 0 - - > 2< 3670016 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486129664 - | - - + - > 5< - - + - | 0 - | 896 - -> 00000080:00200000:1.0:1734016138.440790:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633af8 00000080:00200000:1.0:1734016138.440791:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440792:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440796:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440802:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440807:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440811:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440817:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440819:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3674112 00000080:00200000:1.0:1734016138.440820:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 897 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440828:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3674112, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440830:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440831:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5d4c0 1fffff00000805 3:0 ffff8800a8633bd0 897 lru 00000080:00200000:1.0:1734016138.440835:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633c30 897: 1< 0 - - > 2< 3674112 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486125568 - | - - + - > 5< - - + - | 0 - | 897 - -> 00000080:00200000:1.0:1734016138.440836:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633bd0 00000080:00200000:1.0:1734016138.440837:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440837:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440841:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440845:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440850:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440856:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440861:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440863:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3678208 00000080:00200000:1.0:1734016138.440864:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 898 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440871:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3678208, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440873:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440875:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a555c0 1fffff00000805 3:0 ffff8800a8633ca8 898 lru 00000080:00200000:1.0:1734016138.440879:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633d08 898: 1< 0 - - > 2< 3678208 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486121472 - | - - + - > 5< - - + - | 0 - | 898 - -> 00000080:00200000:1.0:1734016138.440880:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633ca8 00000080:00200000:1.0:1734016138.440880:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440881:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440885:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440890:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440895:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440900:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440906:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440908:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3682304 00000080:00200000:1.0:1734016138.440910:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 899 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440919:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3682304, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440921:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440923:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5c880 1fffff00000805 3:0 ffff8800a8633d80 899 lru 00000080:00200000:1.0:1734016138.440927:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633de0 899: 1< 0 - - > 2< 3682304 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486117376 - | - - + - > 5< - - + - | 0 - | 899 - -> 00000080:00200000:1.0:1734016138.440928:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633d80 00000080:00200000:1.0:1734016138.440929:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440929:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440933:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440937:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440942:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440946:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.440951:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.440953:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3686400 00000080:00200000:1.0:1734016138.440955:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 900 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.440970:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3686400, len 4096, copied 4096 00000080:00200000:1.0:1734016138.440972:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a8633e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.440974:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002afc800 1fffff00000805 3:0 ffff8800a8633e58 900 lru 00000080:00200000:1.0:1734016138.440977:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a8633eb8 900: 1< 0 - - > 2< 3686400 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486113280 - | - - + - > 5< - - + - | 0 - | 900 - -> 00000080:00200000:1.0:1734016138.440978:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a8633e58 00000080:00200000:1.0:1734016138.440979:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.440980:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.440984:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440989:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440994:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.440998:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441003:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441006:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3690496 00000080:00200000:1.0:1734016138.441007:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 901 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441017:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3690496, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441019:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441021:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44040 1fffff00000805 3:0 ffff8800a7055000 901 lru 00000080:00200000:1.0:1734016138.441025:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055060 901: 1< 0 - - > 2< 3690496 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486109184 - | - - + - > 5< - - + - | 0 - | 901 - -> 00000080:00200000:1.0:1734016138.441026:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055000 00000080:00200000:1.0:1734016138.441027:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441027:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441031:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441035:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441040:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441046:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441051:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441053:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3694592 00000080:00200000:1.0:1734016138.441054:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 902 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441061:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3694592, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441063:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a70550d8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441064:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a52a80 1fffff00000805 3:0 ffff8800a70550d8 902 lru 00000080:00200000:1.0:1734016138.441068:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055138 902: 1< 0 - - > 2< 3694592 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486105088 - | - - + - > 5< - - + - | 0 - | 902 - -> 00000080:00200000:1.0:1734016138.441069:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a70550d8 00000080:00200000:1.0:1734016138.441070:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441071:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441074:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441078:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441083:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441088:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441093:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441095:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3698688 00000080:00200000:1.0:1734016138.441096:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 903 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441103:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3698688, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441105:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a70551b0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441107:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5ff00 1fffff00000805 3:0 ffff8800a70551b0 903 lru 00000080:00200000:1.0:1734016138.441110:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055210 903: 1< 0 - - > 2< 3698688 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486100992 - | - - + - > 5< - - + - | 0 - | 903 - -> 00000080:00200000:1.0:1734016138.441111:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a70551b0 00000080:00200000:1.0:1734016138.441112:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441113:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441126:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441131:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441136:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441140:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441147:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441150:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3702784 00000080:00200000:1.0:1734016138.441152:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 904 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441164:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3702784, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441167:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055288[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441169:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5db00 1fffff00000825 2:0 ffff8800a7055288 904 lru 00000080:00200000:1.0:1734016138.441172:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a70552e8 904: 1< 0 - - > 2< 3702784 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486096896 - | - - + - > 5< - - + - | 0 - | 904 - -> 00000080:00200000:1.0:1734016138.441173:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055288 00000080:00200000:1.0:1734016138.441174:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441175:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441180:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441185:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441190:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441196:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441203:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441206:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3706880 00000080:00200000:1.0:1734016138.441208:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 905 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441219:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3706880, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441222:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055360[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441224:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a51500 1fffff00000805 3:0 ffff8800a7055360 905 lru 00000080:00200000:1.0:1734016138.441228:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a70553c0 905: 1< 0 - - > 2< 3706880 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486092800 - | - - + - > 5< - - + - | 0 - | 905 - -> 00000080:00200000:1.0:1734016138.441229:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055360 00000080:00200000:1.0:1734016138.441229:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441230:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441235:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441240:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441245:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441250:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441257:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441259:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3710976 00000080:00200000:1.0:1734016138.441261:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 906 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441271:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3710976, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441274:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055438[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441275:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002afb140 1fffff00000805 3:0 ffff8800a7055438 906 lru 00000080:00200000:1.0:1734016138.441279:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055498 906: 1< 0 - - > 2< 3710976 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486088704 - | - - + - > 5< - - + - | 0 - | 906 - -> 00000080:00200000:1.0:1734016138.441280:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055438 00000080:00200000:1.0:1734016138.441281:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441282:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441287:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441292:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441296:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441302:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441308:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441311:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3715072 00000080:00200000:1.0:1734016138.441313:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 907 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441322:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3715072, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441325:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055510[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441326:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a790c0 1fffff00000805 3:0 ffff8800a7055510 907 lru 00000080:00200000:1.0:1734016138.441330:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055570 907: 1< 0 - - > 2< 3715072 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486084608 - | - - + - > 5< - - + - | 0 - | 907 - -> 00000080:00200000:1.0:1734016138.441331:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055510 00000080:00200000:1.0:1734016138.441332:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441333:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441337:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441342:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441346:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441351:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441357:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441360:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3719168 00000080:00200000:1.0:1734016138.441363:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 908 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441373:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3719168, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441376:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a70555e8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441378:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44e40 1fffff00000805 3:0 ffff8800a70555e8 908 lru 00000080:00200000:1.0:1734016138.441381:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055648 908: 1< 0 - - > 2< 3719168 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486080512 - | - - + - > 5< - - + - | 0 - | 908 - -> 00000080:00200000:1.0:1734016138.441382:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a70555e8 00000080:00200000:1.0:1734016138.441383:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441384:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441389:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441395:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441399:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441404:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441410:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441413:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3723264 00000080:00200000:1.0:1734016138.441415:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 909 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441425:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3723264, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441427:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a70556c0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441429:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a70540 1fffff00000805 3:0 ffff8800a70556c0 909 lru 00000080:00200000:1.0:1734016138.441432:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055720 909: 1< 0 - - > 2< 3723264 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486076416 - | - - + - > 5< - - + - | 0 - | 909 - -> 00000080:00200000:1.0:1734016138.441433:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a70556c0 00000080:00200000:1.0:1734016138.441434:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441435:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441440:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441444:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441449:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441455:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441462:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441465:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3727360 00000080:00200000:1.0:1734016138.441467:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 910 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441475:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3727360, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441478:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055798[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441480:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5a680 1fffff00000805 3:0 ffff8800a7055798 910 lru 00000080:00200000:1.0:1734016138.441484:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a70557f8 910: 1< 0 - - > 2< 3727360 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486072320 - | - - + - > 5< - - + - | 0 - | 910 - -> 00000080:00200000:1.0:1734016138.441485:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055798 00000080:00200000:1.0:1734016138.441486:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441487:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441491:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441496:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441500:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441505:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441512:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441514:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3731456 00000080:00200000:1.0:1734016138.441516:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 911 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441525:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3731456, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441528:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055870[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441529:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a518c0 1fffff00000805 3:0 ffff8800a7055870 911 lru 00000080:00200000:1.0:1734016138.441533:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a70558d0 911: 1< 0 - - > 2< 3731456 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486068224 - | - - + - > 5< - - + - | 0 - | 911 - -> 00000080:00200000:1.0:1734016138.441534:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055870 00000080:00200000:1.0:1734016138.441535:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441536:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441540:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441545:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441549:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441554:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441559:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441561:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3735552 00000080:00200000:1.0:1734016138.441563:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 912 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441572:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3735552, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441575:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055948[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441576:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a51ec0 1fffff00000805 3:0 ffff8800a7055948 912 lru 00000080:00200000:1.0:1734016138.441580:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a70559a8 912: 1< 0 - - > 2< 3735552 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486064128 - | - - + - > 5< - - + - | 0 - | 912 - -> 00000080:00200000:1.0:1734016138.441581:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055948 00000080:00200000:1.0:1734016138.441582:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441583:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441587:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441591:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441596:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441601:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441607:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441610:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3739648 00000080:00200000:1.0:1734016138.441611:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 913 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441620:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3739648, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441622:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055a20[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441624:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a41fc0 1fffff00000805 3:0 ffff8800a7055a20 913 lru 00000080:00200000:1.0:1734016138.441627:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055a80 913: 1< 0 - - > 2< 3739648 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486060032 - | - - + - > 5< - - + - | 0 - | 913 - -> 00000080:00200000:1.0:1734016138.441628:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055a20 00000080:00200000:1.0:1734016138.441629:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441630:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.441636:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441641:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441647:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.441652:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.441659:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.441661:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 3743744 00000080:00200000:1.0:1734016138.441663:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 914 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.441674:0:10402:0:(rw26.c:1006:ll_write_end()) pos 3743744, len 4096, copied 4096 00000080:00200000:1.0:1734016138.441677:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055af8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.441679:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a44f80 1fffff00000805 3:0 ffff8800a7055af8 914 lru 00000080:00200000:1.0:1734016138.441683:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055b58 914: 1< 0 - - > 2< 3743744 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 486055936 - | - - + - > 5< - - + - | 0 - | 914 - -> 00000080:00200000:1.0:1734016138.441684:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055af8 00000080:00200000:1.0:1734016138.441685:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.441686:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000080:00200000:1.0:1734016138.449402:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 1019 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.449420:0:10402:0:(rw26.c:1006:ll_write_end()) pos 4173824, len 4096, copied 4096 00000080:00200000:1.0:1734016138.449424:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055bd0[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.449426:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5e380 1fffff00000805 3:0 ffff8800a7055bd0 1019 lru 00000080:00200000:1.0:1734016138.449429:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055c30 1019: 1< 0 - - > 2< 4173824 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 485625856 - | - - + - > 5< - - + - | 0 - | 1019 - -> 00000080:00200000:1.0:1734016138.449430:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055bd0 00000080:00200000:1.0:1734016138.449431:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.449432:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.449440:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449447:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449451:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449456:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.449464:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.449467:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4177920 00000080:00200000:1.0:1734016138.449469:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 1020 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.449480:0:10402:0:(rw26.c:1006:ll_write_end()) pos 4177920, len 4096, copied 4096 00000080:00200000:1.0:1734016138.449482:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055ca8[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.449484:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5cb00 1fffff00000805 3:0 ffff8800a7055ca8 1020 lru 00000080:00200000:1.0:1734016138.449488:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055d08 1020: 1< 0 - - > 2< 4177920 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 485621760 - | - - + - > 5< - - + - | 0 - | 1020 - -> 00000080:00200000:1.0:1734016138.449489:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055ca8 00000080:00200000:1.0:1734016138.449490:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.449491:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.449495:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449501:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449506:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449511:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.449518:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.449521:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4182016 00000080:00200000:1.0:1734016138.449523:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 1021 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.449532:0:10402:0:(rw26.c:1006:ll_write_end()) pos 4182016, len 4096, copied 4096 00000080:00200000:1.0:1734016138.449535:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055d80[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.449537:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002b8c540 1fffff00000805 3:0 ffff8800a7055d80 1021 lru 00000080:00200000:1.0:1734016138.449541:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055de0 1021: 1< 0 - - > 2< 4182016 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 485617664 - | - - + - > 5< - - + - | 0 - | 1021 - -> 00000080:00200000:1.0:1734016138.449542:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055d80 00000080:00200000:1.0:1734016138.449543:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.449544:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.449548:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449553:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449558:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449563:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.449568:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.449570:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4186112 00000080:00200000:1.0:1734016138.449572:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 1022 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.449582:0:10402:0:(rw26.c:1006:ll_write_end()) pos 4186112, len 4096, copied 4096 00000080:00200000:1.0:1734016138.449584:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7055e58[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.449586:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7dec0 1fffff00000825 2:0 ffff8800a7055e58 1022 lru 00000080:00200000:1.0:1734016138.449589:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7055eb8 1022: 1< 0 - - > 2< 4186112 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 485613568 - | - - + - > 5< - - + - | 0 - | 1022 - -> 00000080:00200000:1.0:1734016138.449591:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7055e58 00000080:00200000:1.0:1734016138.449591:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.449592:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.449596:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449601:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449606:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449610:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.449615:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.449618:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4190208 00000080:00200000:1.0:1734016138.449619:0:10402:0:(rw26.c:819:ll_write_begin()) Writing 1023 of 0 to 4096 bytes 00000080:00200000:1.0:1734016138.449630:0:10402:0:(rw26.c:1006:ll_write_end()) pos 4190208, len 4096, copied 4096 00000080:00200000:1.0:1734016138.449632:0:10402:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71de000[3 ffff8800a9a3dc38 2 1 ffff8800a86824e0] 00000080:00200000:1.0:1734016138.449634:0:10402:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5a8c0 1fffff00000805 3:0 ffff8800a71de000 1023 lru 00000080:00200000:1.0:1734016138.449637:0:10402:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71de060 1023: 1< 0 - - > 2< 4190208 0 0 0x0 0x0 | ffff8800aa644758 ffff8800a9b72b60 > 3< 0 0 > 4< 0 0 8 485609472 - | - - + - > 5< - - + - | 0 - | 1023 - -> 00000080:00200000:1.0:1734016138.449638:0:10402:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71de000 00000080:00200000:1.0:1734016138.449639:0:10402:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016138.449640:0:10402:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 00000008:00010000:1.0:1734016138.449643:0:10402:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449648:0:10402:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3d80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449653:0:10402:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449658:0:10402:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016138.449664:0:10402:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afdec248, count 1, dirtied 1 00000080:00200000:1.0:1734016138.449681:0:10402:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4194304 00000080:00200000:1.0:1734016138.449685:0:10402:0:(file.c:5185:ll_fsync()) VFS Op:inode=[0x200000401:0xa:0x0](ffff8800afdec0c8), start 0, end 4194303, datasync 0 00000080:00200000:1.0:1734016138.449704:0:10402:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016138.449711:0:10402:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1734016138.449723:0:10402:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.449731:0:10402:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016138.450302:0:10402:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016138.451920:0:10402:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800a7443800 - object 0x240000400:6 offset 0<>4194304 00000100:00100000:1.0:1734016138.452002:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335005184:192.168.201.104@tcp:4:dd.0 00000008:00010000:1.0:1734016138.687890:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016138.688366:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3900 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016138.688387:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7443800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335005184:192.168.201.104@tcp:4:dd.0 00010000:00010000:1.0:1734016139.187153:0:4737:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016139.187163:0:4737:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080010000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.187173:0:4737:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b5200 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016139.187178:0:4737:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480010000 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016139.187197:0:4737:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016139.190500:0:4737:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016139.190549:0:4737:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 2/0,0 mode: --/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.190558:0:4737:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016139.190587:0:4737:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 1/0,0 mode: --/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.190594:0:4737:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5200/0xd17286ca2ee506ec lrc: 0/0,0 mode: --/PW res: [0x240000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993e5a expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016139.190616:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016139.190631:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335007232:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016139.190705:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016139.190707:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016139.190709:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016139.198399:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335007232:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016139.201862:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016139.201868:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee506f3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080010000 nid: local remote: 0x979ba2bc55993e61 expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.201875:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b4d80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016139.201880:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee506f3 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480010000 nid: local remote: 0x979ba2bc55993e61 expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016139.201897:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016139.208418:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016139.208457:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee506f3 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993e61 expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.208463:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016139.210688:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee506f3 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993e61 expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.210711:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee506f3 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993e61 expref: -99 pid: 10402 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016139.210734:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016139.210735:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016139.210738:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1734016140.179735:0:10406:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8),name=/ 00000002:00010000:1.0:1734016140.179756:0:10406:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016140.179808:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.179819:0:10406:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.179826:0:10406:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00000100:00100000:1.0:1734016140.179851:0:10406:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7441500 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335009792:192.168.201.104@tcp:101:lfs.0 00000100:00100000:1.0:1734016140.179914:0:10406:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3d80 going to sleep for 11 seconds 00010000:00010000:1.0:1734016140.179934:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016140.185952:0:10406:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7441500 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335009792:192.168.201.104@tcp:101:lfs.0 00010000:00010000:1.0:1734016140.185973:0:10406:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.185982:0:10406:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.185991:0:10406:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.185999:0:10406:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186007:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186013:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186018:0:10406:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016140.186029:0:10406:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800a7441500 x1818247335009792/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1734016156 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.186038:0:10406:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186046:0:10406:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1734016140.186071:0:10406:0:(llite_lib.c:2906:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1734016140.186077:0:10406:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) for lock 0xd17286ca2ee50739 00000080:00010000:1.0:1734016140.186081:0:10406:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) 00000080:00010000:1.0:1734016140.186083:0:10406:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50739 from it ffff8800a77dfbb0 00010000:00010000:1.0:1734016140.186094:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186100:0:10406:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.186116:0:10406:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8), inode mode 41ed mask 1 00000080:00200000:1.0:1734016140.186130:0:10406:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8),name=/ 00000002:00010000:1.0:1734016140.186139:0:10406:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016140.186145:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186151:0:10406:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1734016140.186156:0:10406:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) 00000080:00010000:1.0:1734016140.186158:0:10406:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) for lock 0xd17286ca2ee50739 00000080:00010000:1.0:1734016140.186161:0:10406:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50739 from it ffff8800a77dfba0 00010000:00010000:1.0:1734016140.186163:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186168:0:10406:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.186174:0:10406:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8), inode mode 41ed mask 3 00000080:00200000:1.0:1734016140.186179:0:10406:0:(namei.c:1198:ll_atomic_open()) VFS Op:name=f16b.sanityn, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), file ffff88012b113200, open_flags a141, mode 81b6 opened 0 00000080:00200000:1.0:1734016140.186184:0:10406:0:(namei.c:852:ll_lookup_it()) VFS Op:name=f16b.sanityn, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), intent=open|creat 00000002:00010000:1.0:1734016140.186196:0:10406:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: f16b.sanityn,[0x200000401:0xc:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0120502 00010000:00010000:1.0:1734016140.186204:0:10406:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186233:0:10406:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186238:0:10406:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.186240:0:10406:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee50739 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x979ba2bc55993f17 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.186264:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.186268:0:10406:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00000020:00100000:1.0:1734016140.186276:0:10406:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1734016140.186281:0:10406:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7441500 x1818247335010048/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 528/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:1.0:1734016140.186291:0:10406:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00000100:00100000:1.0:1734016140.186308:0:10406:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7441500 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010048:192.168.201.104@tcp:101:lfs.0 00000100:00100000:1.0:1734016140.186395:0:10406:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3d80 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.198600:0:10406:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7441500 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010048:192.168.201.104@tcp:101:lfs.0 00010000:00010000:1.0:1734016140.198624:0:10406:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200327:0:10406:0:(ldlm_request.c:721:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200338:0:10406:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200346:0:10406:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200354:0:10406:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200366:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200372:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200377:0:10406:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016140.200388:0:10406:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=3 disposition=8000037, status=0 req@ffff8800a7441500 x1818247335010048/t4294968757(4294968757) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'lfs.0' uid:0 gid:0 00000002:00100000:1.0:1734016140.200398:0:10406:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7441500 x1818247335010048/t4294968757(4294968757) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'lfs.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.200408:0:10406:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200417:0:10406:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016140.200438:0:10406:0:(namei.c:1461:ll_create_it()) VFS Op:name=f16b.sanityn, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), intent=open|creat 00000080:00200000:1.0:1734016140.200455:0:10406:0:(llite_lib.c:3172:ll_read_inode2()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00200000:1.0:1734016140.200460:0:10406:0:(llite_lib.c:2906:ll_update_inode()) inode=[0x200000401:0xc:0x0], updating i_size 0 00000080:00200000:1.0:1734016140.200486:0:10406:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1734016140.200491:0:10406:0:(namei.c:1427:ll_create_node()) setting l_ast_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016140.200493:0:10406:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50740 00000080:00200008:1.0:1734016140.200503:0:10406:0:(file.c:862:ll_file_open()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), flags 120501 00000002:00100000:1.0:1734016140.200514:0:10406:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7441500 x1818247335010048/t4294968757(4294968757) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'lfs.0' uid:0 gid:0 00000080:00010000:1.0:1734016140.200574:0:10406:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50740 from it ffff88012e1ad660 00010000:00010000:1.0:1734016140.200578:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.200584:0:10406:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.200603:0:10406:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8), inode mode 81a4 mask 40 00000080:00200080:1.0:1734016140.200632:0:10406:0:(file.c:4494:ll_file_ioctl()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8) cmd=4008669a arg=1faf040 00000002:00010000:1.0:1734016140.200651:0:10406:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 040000002 00010000:00010000:1.0:1734016140.200688:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.200709:0:10406:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00000020:00100000:1.0:1734016140.200717:0:10406:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1734016140.200722:0:10406:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7443100 x1818247335010304/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:1.0:1734016140.200728:0:10406:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00000100:00100000:1.0:1734016140.200746:0:10406:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443100 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010304:192.168.201.104@tcp:101:lfs.0 00000100:00100000:1.0:1734016140.200818:0:10406:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3d80 going to sleep for 11 seconds 00010000:00010000:1.0:1734016140.206585:0:4737:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016140.206591:0:4737:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.206598:0:4737:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff880130978900 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016140.206601:0:4737:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.206607:0:4737:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.206614:0:4737:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.206633:0:4737:0:(vvp_object.c:134:vvp_conf_set()) [0x200000401:0xc:0x0]: losing layout lock 00010000:00010000:1.0:1734016140.206670:0:4737:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.206675:0:4737:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.206718:0:4737:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.206723:0:4737:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee50740 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55993f33 expref: -99 pid: 10406 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.206737:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.206739:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.206741:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016140.206747:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016140.206759:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335010432:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016140.211737:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335010432:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016140.212160:0:10406:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7443100 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010304:192.168.201.104@tcp:101:lfs.0 00010000:00010000:1.0:1734016140.212174:0:10406:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212184:0:10406:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212189:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212194:0:10406:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212200:0:10406:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212205:0:10406:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097aac0 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016140.212209:0:10406:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212215:0:10406:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212220:0:10406:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212225:0:10406:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016140.212229:0:10406:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.212234:0:10406:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097aac0/0xd17286ca2ee50747 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 10406 initiator: MDT0 00000002:00100000:1.0:1734016140.212253:0:10406:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff8800a7443100 x1818247335010304/t4294968758(4294968758) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 648/688 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 301/301 job:'lfs.0' uid:0 gid:0 00000002:00100000:1.0:1734016140.212264:0:10406:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7443100 x1818247335010304/t4294968758(4294968758) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 648/688 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 301/301 job:'lfs.0' uid:0 gid:0 00000080:00200000:1.0:1734016140.212308:0:10406:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016140.212315:0:10406:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016140.212324:0:10406:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016140.212327:0:10406:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016140.212361:0:10406:0:(llite_lib.c:2906:ll_update_inode()) inode=[0x200000401:0xc:0x0], updating i_size 0 00000002:00100000:1.0:1734016140.212374:0:10406:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7443100 x1818247335010304/t4294968758(4294968758) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 600/688 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 301/301 job:'lfs.0' uid:0 gid:0 00000002:00100000:1.0:1734016140.212396:0:10406:0:(mdc_request.c:909:mdc_close()) @@@ matched open req@ffff8800a7443100 x1818247335010304/t4294968758(4294968758) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 600/688 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 301/301 job:'lfs.0' uid:0 gid:0 00000020:00100000:1.0:1734016140.212414:0:10406:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1734016140.212419:0:10406:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7440380 x1818247335010688/t0(0) o35->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016140.212437:0:10406:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440380 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010688:192.168.201.104@tcp:35:lfs.0 00000100:00100000:1.0:1734016140.213637:0:10406:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.218167:0:10406:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440380 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010688:192.168.201.104@tcp:35:lfs.0 00000002:00100000:1.0:1734016140.218195:0:10406:0:(mdc_request.c:819:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8800a7443100 x1818247335010304/t4294968758(4294968758) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 600/688 e 0 to 0 dl 1734016156 ref 2 fl Complete:RQU/204/0 rc 301/301 job:'lfs.0' uid:0 gid:0 00000080:00010000:1.0:1734016140.218220:0:10406:0:(file.c:6669:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xc:0x0](ffff8800afc890c8) 00000002:00010000:1.0:1734016140.218230:0:10406:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: layout flags 0 00010000:00010000:1.0:1734016140.218266:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.218273:0:10406:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00010000:00010000:1.0:1734016140.218280:0:10406:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10406 initiator: MDT0 00000100:00100000:1.0:1734016140.218294:0:10406:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443480 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010816:192.168.201.104@tcp:101:lfs.0 00000100:00100000:1.0:1734016140.219957:0:10406:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.224044:0:10406:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7443480 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335010816:192.168.201.104@tcp:101:lfs.0 00010000:00010000:1.0:1734016140.224069:0:10406:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.224080:0:10406:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1734016140.224102:0:10406:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8800a7443480 x1818247335010816/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 376/424 e 0 to 0 dl 1734016156 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.224111:0:10406:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00000002:00010000:1.0:1734016140.224120:0:10406:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.224129:0:10406:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 16 res 8589935617/12 (0 0) 00000080:00010000:1.0:1734016140.224149:0:10406:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee5074e 00000080:00010000:1.0:1734016140.224154:0:10406:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee5074e from it ffff8800a77dfbf0 00010000:00010000:1.0:1734016140.224157:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.224163:0:10406:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.224180:0:10406:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.224187:0:10406:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016140.224192:0:10406:0:(file.c:6553:ll_layout_lock_set()) ### file [0x200000401:0xc:0x0](ffff8800afc890c8) being reconfigured ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016140.224223:0:10406:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1734016140.224226:0:10406:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.224231:0:10406:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00000080:00200008:1.0:1734016140.224272:0:10406:0:(file.c:427:ll_file_release()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), flags 100001 00000002:00100000:1.0:1734016140.224292:0:10406:0:(mdc_request.c:909:mdc_close()) @@@ matched open req@ffff8800a7441500 x1818247335010048/t4294968757(4294968757) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1734016156 ref 1 fl Complete:RPQU/204/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 00000020:00100000:1.0:1734016140.224313:0:10406:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1734016140.224318:0:10406:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7443480 x1818247335011072/t0(0) o35->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016140.224335:0:10406:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443480 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335011072:192.168.201.104@tcp:35:lfs.0 00000100:00100000:1.0:1734016140.225348:0:10406:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.229857:0:10406:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7443480 pname:cluuid:pid:xid:nid:opc:job lfs:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10406:1818247335011072:192.168.201.104@tcp:35:lfs.0 00000002:00100000:1.0:1734016140.229885:0:10406:0:(mdc_request.c:819:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8800a7441500 x1818247335010048/t4294968757(4294968757) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1734016156 ref 1 fl Complete:RQU/204/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 00010000:00010000:1.0:1734016140.388290:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016140.388299:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee5076a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080010000 nid: local remote: 0x979ba2bc55993fa3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.388307:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a21c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016140.388311:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee5076a lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480010000 nid: local remote: 0x979ba2bc55993fa3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016140.388329:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016140.388339:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016140.388373:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee5076a lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993fa3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.388378:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.388394:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee5076a lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993fa3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.388399:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee5076a lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993fa3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.388418:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.388419:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.388421:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016140.388433:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016140.388446:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335013376:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016140.397944:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335013248:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.397967:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.397976:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.397984:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401686:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401729:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401749:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016140.401760:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401769:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401775:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401798:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401805:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x80000000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.401873:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016140.401896:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440000 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335013504:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016140.401930:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.404381:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335013376:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016140.405223:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016140.405227:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3600/0xd17286ca2ee50771 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080010000 nid: local remote: 0x979ba2bc55993faa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.405235:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3600 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016140.405239:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3600/0xd17286ca2ee50771 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480010000 nid: local remote: 0x979ba2bc55993faa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016140.405255:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016140.405268:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016140.405297:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3600/0xd17286ca2ee50771 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993faa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.405302:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.405510:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3600/0xd17286ca2ee50771 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993faa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.405517:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3600/0xd17286ca2ee50771 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480010020 nid: local remote: 0x979ba2bc55993faa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.405544:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.405545:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.405547:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016140.418036:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440000 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335013504:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.418072:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.418095:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.418103:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.420490:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.420500:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.420525:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016140.420538:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.420548:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.420553:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016140.438149:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1734016140.438157:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774dd58 (ffff8800a8711a28/0) at cl_lock_release():233 00000020:00010000:1.0:1734016140.438160:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774dd58 (ffff8800a8711a28/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016140.438164:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a876bb70 (ffff8800a8711368/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016140.438176:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.438190:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016140.438206:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a876bbb0 (ffff8800a8711f38/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016140.438211:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.438217:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016140.438225:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774dd58 (ffff8800a8711a28/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016140.438227:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a876bb70 (ffff8800a8711a28/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016140.438232:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a876bbb0 (ffff8800a8711a28/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016140.438249:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016140.438266:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 0, valid attrs a168, mode 0, uid -1343863320, gid -30720, new size 0, atime 1734016140.000000000, mtime 1734016140.366000000, ctime 1734016140.366000000, rc 0 00000080:00200000:1.0:1734016140.438415:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8),name=/ 00000002:00010000:1.0:1734016140.438434:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016140.438492:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016140.438499:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016140.438509:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016140.438531:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440e00 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014016:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016140.438608:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.453599:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440e00 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014016:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.453625:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453636:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453646:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453655:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453664:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453671:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453676:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016140.453686:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800a7440e00 x1818247335014016/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1734016156 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.453719:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453731:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1734016140.453760:0:10407:0:(llite_lib.c:2906:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1734016140.453766:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) for lock 0xd17286ca2ee5079b 00000080:00010000:1.0:1734016140.453769:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) 00000080:00010000:1.0:1734016140.453772:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee5079b from it ffff8800a872fbb0 00010000:00010000:1.0:1734016140.453774:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453781:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.453803:0:10407:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8), inode mode 41ed mask 1 00000080:00200000:1.0:1734016140.453825:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8),name=/ 00000002:00010000:1.0:1734016140.453838:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016140.453844:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453851:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1734016140.453856:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) 00000080:00010000:1.0:1734016140.453858:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) for lock 0xd17286ca2ee5079b 00000080:00010000:1.0:1734016140.453861:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee5079b from it ffff8800a872fba0 00010000:00010000:1.0:1734016140.453864:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453869:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.453876:0:10407:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8), inode mode 41ed mask 3 00000080:00200000:1.0:1734016140.453881:0:10407:0:(namei.c:1198:ll_atomic_open()) VFS Op:name=f16b.sanityn.fsxgood, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), file ffff88012b113e00, open_flags 8242, mode 81b6 opened 0 00000080:00200000:1.0:1734016140.453888:0:10407:0:(namei.c:852:ll_lookup_it()) VFS Op:name=f16b.sanityn.fsxgood, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), intent=open|creat 00000002:00010000:1.0:1734016140.453902:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: f16b.sanityn.fsxgood,[0x200000401:0xe:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101103 00010000:00010000:1.0:1734016140.453912:0:10407:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453949:0:10407:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.453954:0:10407:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.453956:0:10407:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee5079b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x979ba2bc55994005 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.454004:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016140.454010:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000020:00100000:1.0:1734016140.454019:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1734016140.454024:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7440e00 x1818247335014272/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:1.0:1734016140.454035:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016140.454054:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440e00 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014272:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016140.454187:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.460987:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440e00 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014272:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.461025:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461055:0:10407:0:(ldlm_request.c:721:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xe:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461062:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461072:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461089:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461100:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461106:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.461112:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016140.461122:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8800a7440e00 x1818247335014272/t4294968767(4294968767) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 664/664 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00100000:1.0:1734016140.461133:0:10407:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7440e00 x1818247335014272/t4294968767(4294968767) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 664/664 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.461145:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016140.461154:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.461163:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/14 (0 0) 00000080:00200000:1.0:1734016140.461187:0:10407:0:(namei.c:1461:ll_create_it()) VFS Op:name=f16b.sanityn.fsxgood, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), intent=open|creat 00000080:00200000:1.0:1734016140.461211:0:10407:0:(llite_lib.c:3172:ll_read_inode2()) VFS Op:inode=[0x200000401:0xe:0x0](ffff8800afc898c8) 00000080:00200000:1.0:1734016140.461279:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xe:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1734016140.461283:0:10407:0:(namei.c:1427:ll_create_node()) setting l_ast_data to inode [0x200000401:0xe:0x0](ffff8800afc898c8) 00000080:00010000:1.0:1734016140.461285:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xe:0x0](ffff8800afc898c8) for lock 0xd17286ca2ee507a2 00000080:00200008:1.0:1734016140.461299:0:10407:0:(file.c:862:ll_file_open()) START file f16b.sanityn.fsxgood:[0x200000401:0xe:0x0](ffff8800afc898c8), flags 101102 00000002:00100000:1.0:1734016140.461313:0:10407:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7440e00 x1818247335014272/t4294968767(4294968767) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 592/664 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000080:00010000:1.0:1734016140.461404:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee507a2 from it ffff88012e1add80 00010000:00010000:1.0:1734016140.461408:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.461414:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016140.461438:0:10407:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000401:0xe:0x0](ffff8800afc898c8), inode mode 81a4 mask 40 00000080:00200000:1.0:1734016140.461515:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8),name=/ 00000002:00010000:1.0:1734016140.461527:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016140.461567:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016140.461573:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016140.461580:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016140.461599:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440700 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014400:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016140.461646:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3600 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.475814:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440700 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014400:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.475845:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475855:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475866:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475875:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475887:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475895:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475901:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016140.475913:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800a7440700 x1818247335014400/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1734016156 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.475925:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475938:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1734016140.475973:0:10407:0:(llite_lib.c:2906:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1734016140.475982:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) for lock 0xd17286ca2ee507a9 00000080:00010000:1.0:1734016140.475986:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) 00000080:00010000:1.0:1734016140.475989:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee507a9 from it ffff8800a872fbb0 00010000:00010000:1.0:1734016140.475992:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.475997:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.476022:0:10407:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8), inode mode 41ed mask 1 00000080:00200000:1.0:1734016140.476045:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8),name=/ 00000002:00010000:1.0:1734016140.476058:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016140.476064:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.476075:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1734016140.476090:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) 00000080:00010000:1.0:1734016140.476092:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8800afde80c8) for lock 0xd17286ca2ee507a9 00000080:00010000:1.0:1734016140.476095:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee507a9 from it ffff8800a872fba0 00010000:00010000:1.0:1734016140.476097:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.476101:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1734016140.476106:0:10407:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8800afde80c8), inode mode 41ed mask 3 00000080:00200000:1.0:1734016140.476112:0:10407:0:(namei.c:1198:ll_atomic_open()) VFS Op:name=f16b.sanityn.fsxlog, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), file ffff88012b113c80, open_flags 8241, mode 81b6 opened 0 00000080:00200000:1.0:1734016140.476121:0:10407:0:(namei.c:852:ll_lookup_it()) VFS Op:name=f16b.sanityn.fsxlog, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), intent=open|creat 00000002:00010000:1.0:1734016140.476138:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: f16b.sanityn.fsxlog,[0x200000401:0xf:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:1.0:1734016140.476148:0:10407:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.476186:0:10407:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.476191:0:10407:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.476193:0:10407:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee507a9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x979ba2bc55994036 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.477492:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016140.477501:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000020:00100000:1.0:1734016140.477512:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1734016140.477519:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7440700 x1818247335014656/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:1.0:1734016140.477528:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016140.477550:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440700 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014656:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016140.480267:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3600 going to sleep for 11 seconds 00000100:00100000:1.0:1734016140.498780:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7440700 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014656:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.498808:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498839:0:10407:0:(ldlm_request.c:721:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xf:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498846:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498855:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498864:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498925:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498932:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016140.498938:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016140.498948:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8800a7440700 x1818247335014656/t4294968768(4294968768) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 664/664 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00100000:1.0:1734016140.498959:0:10407:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7440700 x1818247335014656/t4294968768(4294968768) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 664/664 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016140.498971:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016140.498981:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.498991:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/15 (0 0) 00000080:00200000:1.0:1734016140.499014:0:10407:0:(namei.c:1461:ll_create_it()) VFS Op:name=f16b.sanityn.fsxlog, dir=[0x200000007:0x1:0x0](ffff8800afde80c8), intent=open|creat 00000080:00200000:1.0:1734016140.499039:0:10407:0:(llite_lib.c:3172:ll_read_inode2()) VFS Op:inode=[0x200000401:0xf:0x0](ffff8800afc8a0c8) 00000080:00200000:1.0:1734016140.499157:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xf:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1734016140.499162:0:10407:0:(namei.c:1427:ll_create_node()) setting l_ast_data to inode [0x200000401:0xf:0x0](ffff8800afc8a0c8) 00000080:00010000:1.0:1734016140.499165:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xf:0x0](ffff8800afc8a0c8) for lock 0xd17286ca2ee507b0 00000080:00200008:1.0:1734016140.499181:0:10407:0:(file.c:862:ll_file_open()) START file f16b.sanityn.fsxlog:[0x200000401:0xf:0x0](ffff8800afc8a0c8), flags 101101 00000002:00100000:1.0:1734016140.499194:0:10407:0:(mdc_request.c:798:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8800a7440700 x1818247335014656/t4294968768(4294968768) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 592/664 e 0 to 0 dl 1734016156 ref 2 fl Complete:RPQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000080:00010000:1.0:1734016140.499284:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee507b0 from it ffff88012e1add80 00010000:00010000:1.0:1734016140.499288:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016140.499295:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016140.499316:0:10407:0:(file.c:6234:ll_inode_permission()) VFS Op:inode=[0x200000401:0xf:0x0](ffff8800afc8a0c8), inode mode 81a4 mask 40 00000080:00200008:1.0:1734016140.944508:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 0, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 0, atime 1734016140.000000000, mtime 1734016140.000000000, ctime 1734016140.000000000 00000020:00100000:1.0:1734016140.944590:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016140.944600:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7440a80 x1818247335014912/t0(0) o36->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016140.944621:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7440a80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335014912:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016140.953317:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3900 going to sleep for 11 seconds 00010000:00010000:1.0:1734016140.970887:0:4737:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016140.970894:0:4737:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080030000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.970903:0:4737:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1d40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016140.970907:0:4737:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480030000 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016140.970924:0:4737:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016140.970934:0:4737:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016140.970966:0:4737:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.970976:0:4737:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.971010:0:4737:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.971017:0:4737:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee5078d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993ff0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.971045:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.971047:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.971048:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016140.977944:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335015040:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.977970:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.977979:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.977987:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979358:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979366:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979390:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016140.979399:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979407:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979412:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979438:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979443:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x80000000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.979468:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016140.979485:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7442d80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335015296:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016140.979530:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3180 going to sleep for 11 seconds 00010000:00010000:1.0:1734016140.990213:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016140.990219:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080030000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.990231:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097aac0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016140.990235:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480030000 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016140.990252:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016140.990263:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016140.990295:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.990300:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016140.990323:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.990329:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50794 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993ff7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.990351:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.990352:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016140.990354:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016140.998403:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7442d80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335015296:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016140.998426:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.998436:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016140.998532:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.002230:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.002241:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.002262:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.002273:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.002283:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.002289:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.015922:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1734016141.015929:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774d8a8 (ffff8800a87112d8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.015933:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774d8a8 (ffff8800a87112d8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.015937:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a876b330 (ffff8800a8711008/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.015947:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.015959:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.015975:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a876b370 (ffff8800a87113f8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.015979:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.015986:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.015995:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774d8a8 (ffff8800a87112d8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.015997:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a876b330 (ffff8800a87112d8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.016001:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a876b370 (ffff8800a87112d8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.016020:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.016036:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 0, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 0, atime 1734016140.000000000, mtime 1734016140.943000000, ctime 1734016140.943000000, rc 0 00000080:00200008:1.0:1734016141.016058:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 0, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 100000, atime 1734016141.000000000, mtime 1734016141.000000000, ctime 1734016141.000000000 00000020:00100000:1.0:1734016141.018580:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016141.018594:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7442300 x1818247335015936/t0(0) o36->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016141.018616:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7442300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335015936:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016141.020068:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3180 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.027344:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7442300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335015936:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016141.027383:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016141.027387:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.027402:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.027417:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016141.027430:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.027435:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be31b0, osc lock ffff8800a86724c0, flags 0x0 00000008:00010000:1.0:1734016141.027438:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.027439:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be31f0, osc lock ffff8800a8672d10, flags 0x0 00010000:00010000:1.0:1734016141.027458:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027470:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027479:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027485:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027490:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027500:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027506:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027512:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027518:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.027523:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.042631:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 100000 i_blocks 0 00000020:00010000:1.0:1734016141.042636:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774d8a8 (ffff8800a87112d8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.042639:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774d8a8 (ffff8800a87112d8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.042643:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be31b0 (ffff8800a87113f8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.042651:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.042662:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.042675:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be31f0 (ffff8800a8711008/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.042679:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.042685:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.042714:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774d8a8 (ffff8800a87112d8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.042715:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be31b0 (ffff8800a87112d8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.042719:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be31f0 (ffff8800a87112d8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.042733:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.042745:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 100000, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 100000, atime 1734016141.000000000, mtime 1734016141.015000000, ctime 1734016141.015000000, rc 0 00000080:00200008:1.0:1734016141.042775:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016141.042780:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016141.042794:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016141.042804:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016141.042836:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee507c5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016141.042841:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee507c5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016141.042849:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978d80/0xd17286ca2ee507c5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016141.042864:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443b80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335016320:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.042922:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3180 going to sleep for 11 seconds 00010000:00010000:1.0:1734016141.097830:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.097836:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.097844:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a0fc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.097849:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480020000 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.097872:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.097888:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.097933:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480020020 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.097938:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.100986:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480020020 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.101001:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee507b7 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480020020 nid: local remote: 0x979ba2bc55994067 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.101040:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.101042:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.101044:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.101055:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016141.101087:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335017728:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016141.111539:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335017728:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016141.123316:0:4737:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.123322:0:4737:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080030000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.123329:0:4737:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8801309798c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.123333:0:4737:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480030000 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.123352:0:4737:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.123362:0:4737:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.123391:0:4737:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.123396:0:4737:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.123419:0:4737:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.123425:0:4737:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee507be lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc5599406e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.123446:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.123447:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.123448:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.123454:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016141.123464:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335018368:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016141.133994:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335018368:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016141.150163:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.150177:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee507cc lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800420000030000 nid: local remote: 0x979ba2bc55994098 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.150186:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.150190:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee507cc lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800428400030000 nid: local remote: 0x979ba2bc55994098 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.150212:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.150223:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.150261:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee507cc lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x804c69400030020 nid: local remote: 0x979ba2bc55994098 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.150266:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.150298:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee507cc lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x804c69400030020 nid: local remote: 0x979ba2bc55994098 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.150303:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee507cc lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x804c69400030020 nid: local remote: 0x979ba2bc55994098 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.150331:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.150333:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.150334:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.161911:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.161919:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee507d3 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000030000 nid: local remote: 0x979ba2bc559940a6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.161928:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a0240 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.161932:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee507d3 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400030000 nid: local remote: 0x979ba2bc559940a6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.161949:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.161963:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.162000:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee507d3 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400030020 nid: local remote: 0x979ba2bc559940a6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.162007:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.162042:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee507d3 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400030020 nid: local remote: 0x979ba2bc559940a6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.162049:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee507d3 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400030020 nid: local remote: 0x979ba2bc559940a6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.162093:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.162094:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.162096:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.166459:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335019264:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.166485:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.166494:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.166501:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.167829:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.167837:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.167864:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.167877:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.167887:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.167893:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.175189:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1734016141.175195:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.175198:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.175202:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8602030 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.175209:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.175220:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.175231:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8602070 (ffff8800a8711518/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.175235:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.175247:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.175254:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.175255:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8602030 (ffff8800a8711b48/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.175259:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8602070 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.175269:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.175280:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 0, valid attrs 2168, mode 0, uid -1248067584, gid -30720, new size 0, atime 1734016141.000000000, mtime 1734016141.133000000, ctime 1734016141.133000000, rc 0 00000080:00200000:1.0:1734016141.175300:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016141.175305:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.175313:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.175321:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 1023] 00000008:00010000:1.0:1734016141.175330:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.175333:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a8672d10, flags 0x0 00010000:00010000:1.0:1734016141.175348:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.175355:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.175378:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.175393:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443800 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335019776:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.175446:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00010000:00010000:1.0:1734016141.180862:0:4737:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.180870:0:4737:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080020000 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.180880:0:4737:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b6400 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.180885:0:4737:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480020000 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.180905:0:4737:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.180914:0:4737:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.180948:0:4737:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480020020 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.180953:0:4737:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.180969:0:4737:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480020020 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.180975:0:4737:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6400/0xd17286ca2ee507da lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480020020 nid: local remote: 0x979ba2bc559940b4 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.181714:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.181716:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.181718:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.183261:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.191903:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335019904:192.168.201.104@tcp:103:ldlm_bl.0 00000020:00010000:1.0:1734016141.191933:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.191935:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.191939:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711518/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.191948:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.191958:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.191969:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.191971:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.191983:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.191999:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 0, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 0, atime 1734016141.000000000, mtime 1734016141.000000000, ctime 1734016141.000000000 00000020:00100000:1.0:1734016141.192040:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800aa629800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016141.192046:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a7441880 x1818247335020288/t0(0) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016141.192065:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335020288:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016141.192119:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3480 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.195766:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7441880 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335020288:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016141.195806:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016141.195811:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.195823:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.195838:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016141.195848:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:1.0:1734016141.195851:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be34b0, osc lock ffff8800a8672d10, flags 0x80000000 00000008:00010000:1.0:1734016141.195853:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:1.0:1734016141.195854:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be34f0, osc lock ffff8800a86724c0, flags 0x80000000 00010000:00010000:1.0:1734016141.195871:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195886:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195897:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195902:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195908:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195925:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195931:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x80000000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.195959:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.195977:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a7443800 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335020416:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.196026:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00010000:00010000:1.0:1734016141.201778:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.201787:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080030000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.201799:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3840 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.201804:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480030000 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.201823:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.201832:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.201864:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.201869:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.201935:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.201941:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee507e1 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc559940bb expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.202715:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.202716:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.202718:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.206999:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a7443800 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335020416:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.207023:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.207032:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.207044:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.210193:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.210202:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.210224:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.210235:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.210245:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.210251:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.215260:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 0 i_blocks 8192 00000020:00010000:1.0:1734016141.215267:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.215270:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.215273:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be34b0 (ffff8800a8711518/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.215283:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.215297:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.215311:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be34f0 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.215316:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.215324:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.215334:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.215336:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be34b0 (ffff8800a8711b48/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.215339:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be34f0 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.215358:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.215371:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 0, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 0, atime 1734016141.000000000, mtime 1734016141.191000000, ctime 1734016141.191000000, rc 0 00000080:00200008:1.0:1734016141.215641:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 0, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 18048, atime 1734016141.000000000, mtime 1734016141.000000000, ctime 1734016141.000000000 00000020:00100000:1.0:1734016141.215724:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800aa629800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016141.215732:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59dfb80 x1818247335021056/t0(0) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016141.215753:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335021056:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016141.215822:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3480 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.221258:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335021056:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016141.221296:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016141.221301:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.221315:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.221331:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016141.221342:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.221346:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be34b0, osc lock ffff8800a86724c0, flags 0x0 00000008:00010000:1.0:1734016141.221349:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.221350:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be34f0, osc lock ffff8800a8672d10, flags 0x0 00010000:00010000:1.0:1734016141.221366:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221377:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221388:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221396:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221402:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221413:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221419:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221430:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221435:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.221440:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.225326:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.225333:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.225335:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.225339:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be34b0 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.225348:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225359:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.225370:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be34f0 (ffff8800a8711518/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.225374:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225380:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.225392:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.225393:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be34b0 (ffff8800a8711b48/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.225397:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be34f0 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.225411:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.225423:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 18048, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 18048, atime 1734016141.000000000, mtime 1734016141.214000000, ctime 1734016141.214000000, rc 0 00000080:00a00000:1.0:1734016141.225465:0:10407:0:(llite_mmap.c:611:ll_file_mmap()) VFS_Op: fid=[0x200000401:0xc:0x0] vma=ffff8800b6f45d80 start=0x7f7d584a8000 end=0x7f7d584ab000 vm_flags=0xfb 00000080:00200000:1.0:1734016141.225475:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.225482:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.225491:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.225493:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be34b0, osc lock ffff8800a8672d10, flags 0x400001002 00000008:00010000:1.0:1734016141.225497:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.225498:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be34f0, osc lock ffff8800a86724c0, flags 0x400001002 00010000:00010000:1.0:1734016141.225511:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225523:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225532:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225538:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225548:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225557:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225565:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225572:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.225583:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.225586:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.225589:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.225591:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be34b0 (ffff8800a8711518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.225594:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be34f0 (ffff8800a8711248/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.225596:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.225598:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be34b0 (ffff8800a8711b48/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.225601:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be34f0 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.225606:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.225659:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 2 00000080:00200000:1.0:1734016141.225665:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.225672:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.225680:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2, 2] 00000008:00010000:1.0:1734016141.225687:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.225730:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.225742:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225757:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (8192 12287) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225765:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225772:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.225777:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.232219:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.232223:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.232226:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.232232:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232240:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.232252:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.232254:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.232268:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.232280:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.232289:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.232299:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2, 2] 00000008:00010000:1.0:1734016141.232308:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.232311:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.232323:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232332:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (8192 12287) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232340:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232345:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232351:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.232375:0:10407:0:(osc_cache.c:783:osc_extent_find()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.232387:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 1, dirtied 1 00000020:00010000:1.0:1734016141.232396:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.232398:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.232399:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.232402:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232408:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.232415:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.232417:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.232423:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.232458:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 3 00000080:00200000:1.0:1734016141.232465:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.232469:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.232474:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [3, 3] 00000008:00010000:1.0:1734016141.232481:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.232483:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.232489:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232496:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (12288 16383) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232502:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232508:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.232512:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.236501:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.236505:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.236507:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.236513:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236521:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.236531:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.236533:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.236544:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.236555:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.236564:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.236574:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [3, 3] 00000008:00010000:1.0:1734016141.236582:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.236585:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.236596:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236604:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (12288 16383) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236612:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236617:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236621:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.236648:0:10407:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3e40 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.236654:0:10407:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3e40 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.236660:0:10407:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.236665:0:10407:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.236675:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 1, dirtied 1 00000020:00010000:1.0:1734016141.236681:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.236682:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.236684:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.236686:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236717:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.236723:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.236724:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.236729:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.236756:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 4 00000080:00200000:1.0:1734016141.236760:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.236764:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.236767:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [4, 4] 00000008:00010000:1.0:1734016141.236771:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.236772:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.236777:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236783:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (16384 20479) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236790:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236795:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236801:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000001:00010000:1.0:1734016141.236811:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.236814:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.236816:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672d10, flags 0x400001002 00000008:00010000:1.0:1734016141.236821:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.236822:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016141.236826:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236832:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236843:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 8/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236849:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236857:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236863:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236868:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236875:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.236885:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.236888:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.236889:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.236891:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff8800a8711248/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.236893:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff8800a8711518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.236896:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.236897:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff8800a8711b48/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.236900:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff8800a8711b48/1) at cl_lock_fini():85 00010000:00010000:1.0:1734016141.236921:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236927:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4096 8191) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236933:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 8/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236938:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 8/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236942:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236957:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.236962:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.243039:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.243043:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.243046:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.243051:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243063:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.243087:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.243089:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.243103:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.243116:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.243130:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.243139:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [4, 4] 00000008:00010000:1.0:1734016141.243150:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.243153:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8801370ce430, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.243166:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243175:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (16384 20479) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243183:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243189:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243194:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000001:00010000:1.0:1734016141.243211:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.243215:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.243217:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016141.243221:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.243223:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a8672d10, flags 0x400001002 00010000:00010000:1.0:1734016141.243228:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243233:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243239:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 8/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243245:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 7/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243253:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243259:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243265:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243270:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.243281:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.243285:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.243287:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.243289:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a8711518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.243291:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a8711248/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.243293:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8680ff8 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.243294:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711b48/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.243297:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711b48/1) at cl_lock_fini():85 00000008:00010000:1.0:1734016141.243314:0:10407:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be3c00 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.243320:0:10407:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3c00 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.243326:0:10407:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.243332:0:10407:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.243344:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 1, dirtied 1 00000020:00010000:1.0:1734016141.243352:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.243354:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.243356:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8801370ce430 (ffff8800a8711518/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.243359:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243365:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.243376:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774db78 (ffff8800a8711b48/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.243378:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8801370ce430 (ffff8800a8711b48/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.243384:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016141.243422:0:10407:0:(file.c:5185:ll_fsync()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8), start 8192, end 20479, datasync 1 00000080:00200000:1.0:1734016141.243427:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.243430:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1734016141.243437:0:10407:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.243445:0:10407:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.243536:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (8192 12287) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016141.243589:0:10407:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dd500 - object 0x280000400:7 offset 8192<>18048 00000080:00200000:1.0:1734016141.248437:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1734016141.248478:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335022336:192.168.201.104@tcp:44:fsx.0 00000100:00100000:1.0:1734016141.248528:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.258413:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8800aa629800: committing for last_committed 4294968784 gen 1 00000100:00100000:1.0:1734016141.258432:0:10407:0:(client.c:3017:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff88012cc33100 x1818247335012736/t4294968764(4294968764) o101->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 608/688 e 0 to 0 dl 1734016156 ref 1 fl Complete:RPQU/204/ffffffff rc 0/-1 job:'fsx.0' uid:0 gid:0 00000100:00100000:1.0:1734016141.258456:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335022336:192.168.201.104@tcp:44:fsx.0 00000080:00200000:1.0:1734016141.258476:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.258486:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1734016141.258539:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016141.258553:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335022464:192.168.201.104@tcp:16:fsx.0 00010000:00010000:1.0:1734016141.264717:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.264731:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50716 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420080030000 nid: local remote: 0x979ba2bc55993eb5 expref: -99 pid: 10404 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.264741:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2f40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.264747:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50716 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428480030000 nid: local remote: 0x979ba2bc55993eb5 expref: -99 pid: 10404 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.264768:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.271277:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xa:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.271335:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50716 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993eb5 expref: -99 pid: 10404 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.271343:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.271364:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50716 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993eb5 expref: -99 pid: 10404 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.271370:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50716 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69480030020 nid: local remote: 0x979ba2bc55993eb5 expref: -99 pid: 10404 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.271390:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.271392:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.271395:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.271420:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335022592:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016141.288831:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335022592:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016141.290523:0:1969:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8800aa629800: committing for last_committed 4294969601 gen 1 00000100:00100000:1.0:1734016141.290606:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335022464:192.168.201.104@tcp:16:fsx.0 00000080:00200008:1.0:1734016141.303556:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 18048 00000080:00200008:1.0:1734016141.303575:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016141.303582:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016141.303601:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016141.303613:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.303625:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016141.303640:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016141.303643:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee507f6 00000080:00010000:1.0:1734016141.303650:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee507f6 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016141.303653:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.303660:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016141.303684:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.303708:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.303724:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.303726:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a86724c0, flags 0x400001002 00000008:00010000:1.0:1734016141.303732:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.303733:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672d10, flags 0x400001002 00010000:00010000:1.0:1734016141.303745:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303753:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303760:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303765:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303775:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303781:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303786:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303791:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.303803:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.303806:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.303808:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.303812:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87112d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.303814:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.303816:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.303818:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff8800a8711518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.303821:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff8800a8711518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.303826:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.303837:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016141.303840:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.303842:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.303846:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.303847:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672d10, flags 0x400001002 00000008:00010000:1.0:1734016141.303849:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.303850:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a86724c0, flags 0x400001002 00010000:00010000:1.0:1734016141.303855:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303861:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303868:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303873:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303883:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303893:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303898:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.303903:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.303909:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.303911:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.303912:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.303914:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.303916:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87112d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.303917:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.303919:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff8800a8711518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.303921:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff8800a8711518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.303924:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.303930:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 18048 00000080:00200000:1.0:1734016141.303963:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016141.303969:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.303974:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.303985:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2, 2] 00000008:00010000:1.0:1734016141.303990:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.303993:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea2b0, osc lock ffff8800a86724c0, flags 0x0 00010000:00010000:1.0:1734016141.304008:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.304015:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [8192->12287] (req 8192->12287) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.304044:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [8192->12287] (req 8192->12287) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.304078:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335022976:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.304401:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3c00 going to sleep for 11 seconds 00010000:00010000:1.0:1734016141.313165:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.313172:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.313179:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097a640 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.313184:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.313205:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.314684:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.315685:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.315723:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.315749:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.315755:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee507e8 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559940c2 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.315779:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.315780:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.315783:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.317632:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8800b588c800: committing for last_committed 4294969597 gen 2 00000100:00100000:1.0:1734016141.317680:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335022976:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.317733:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [8192->12287] (req 8192->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.317742:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [8192->12287] (req 8192->12287) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.317749:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [8192->12287] (req 8192->12287) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.346605:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.346617:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.346640:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.346650:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 18048, setting rss=18048; kms=18048, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.346663:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.346670:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.346703:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.372562:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774d6c8 (ffff8800a8711518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.372568:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774d6c8 (ffff8800a8711518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.372572:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea2b0 (ffff8800a87112d8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016141.372580:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.372591:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.372605:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774d6c8 (ffff8800a8711518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.372607:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea2b0 (ffff8800a8711518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.372621:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.372639:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016141.372643:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016141.372660:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016141.372673:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016141.372729:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016141.372740:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016141.372746:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016141.372766:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335023360:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.372845:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3c00 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.383315:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8800b588c800: committing for last_committed 4294968780 gen 1 00000100:00100000:1.0:1734016141.383419:0:10407:0:(client.c:3017:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8800b59c8380 x1818247335011456/t4294968761(4294968761) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 608/688 e 0 to 0 dl 1734016156 ref 1 fl Complete:RPQU/204/ffffffff rc 0/-1 job:'fsx.0' uid:0 gid:0 00000100:00100000:1.0:1734016141.383432:0:10407:0:(client.c:3017:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8800a7440e00 x1818247335014272/t4294968767(4294968767) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 592/664 e 0 to 0 dl 1734016156 ref 1 fl Complete:RPQU/204/ffffffff rc 0/-1 job:'fsx.0' uid:0 gid:0 00000100:00100000:1.0:1734016141.383439:0:10407:0:(client.c:3017:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8800a7440700 x1818247335014656/t4294968768(4294968768) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 592/664 e 0 to 0 dl 1734016156 ref 1 fl Complete:RPQU/204/ffffffff rc 0/-1 job:'fsx.0' uid:0 gid:0 00000100:00100000:1.0:1734016141.383462:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335023360:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.383480:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.383489:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.383499:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.383509:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.383522:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.383528:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.383533:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016141.383539:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59df800 x1818247335023360/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016157 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016141.383547:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016141.383555:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.383564:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016141.383607:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016141.383612:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50804 00000080:00010000:1.0:1734016141.383616:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016141.383619:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50804 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016141.383621:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.383627:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50804 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994108 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016141.383651:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.383659:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.383672:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.383674:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86724c0, flags 0x400001002 00000008:00010000:1.0:1734016141.383679:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.383681:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a8672d10, flags 0x400001002 00010000:00010000:1.0:1734016141.383720:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.383730:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.383738:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.383743:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.383760:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.383766:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.383812:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016141.383826:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335023488:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.394400:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394418:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394429:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394435:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394440:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.394451:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016141.394487:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394492:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394497:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394503:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097a1c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.394507:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394519:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016141.394523:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.394528:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5080b lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.394564:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335023488:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.444627:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.444641:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.444653:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097bcc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.444658:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.444680:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.444720:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.444759:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.444765:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.446304:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.446315:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee507fd lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559940fa expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.465348:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.465353:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.465358:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1734016141.487525:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.487531:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.487534:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.487538:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a9971db0 (ffff8800a87112d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.487541:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a9971df0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.487543:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.487545:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a9971db0 (ffff8800a8711518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.487548:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a9971df0 (ffff8800a8711518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.487557:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.487573:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016141.487577:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.487583:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.487594:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.487596:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016141.487603:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.487604:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672720, flags 0x400001002 00010000:00010000:1.0:1734016141.487618:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee5082e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.487624:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee5082e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.487663:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50835 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.487673:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50835 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.510251:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 18048 i_blocks 40 00000020:00010000:1.0:1734016141.510256:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.510258:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.510262:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.510266:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87112d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.510268:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8683908 (ffff8800a8711518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.510270:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff8800a8711518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.510273:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff8800a8711518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.510281:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.510288:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 18048 00000002:00010000:1.0:1734016141.510470:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000401:0xc:0x0], intent: getxattr flags 0 00010000:00010000:1.0:1734016141.510503:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 3/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016141.510509:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 3/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016141.510517:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 3/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016141.510537:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335024640:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.510601:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.519296:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335024640:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.519321:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994147 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.519335:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994147 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.519343:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994147 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016141.519352:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff8800b59dea00 x1818247335024640/t0(0) o101->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 568/1176 e 0 to 0 dl 1734016157 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016141.519361:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994147 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.519371:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b3e08000 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00010000:1.0:1734016141.519396:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee5083c 00000080:00010000:1.0:1734016141.519403:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee5083c from it ffff8800a872fc50 00010000:00010000:1.0:1734016141.519405:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994147 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016141.519411:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097b3c0/0xd17286ca2ee5083c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994147 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1734016141.519440:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 18048, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 691702, atime 1734016141.000000000, mtime 1734016141.000000000, ctime 1734016141.000000000 00010000:00010000:1.0:1734016141.519461:0:10407:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.519474:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4480/0xd17286ca2ee50786 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993fe9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.519504:0:10407:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.519509:0:10407:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.519511:0:10407:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6400/0xd17286ca2ee507f6 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc559940f3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1734016141.519530:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800aa629800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016141.519535:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59dea00 x1818247335024896/t0(0) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016141.519554:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335024896:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016141.519600:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016141.540137:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335024896:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016141.540178:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016141.540183:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016141.540195:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016141.540208:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016141.540217:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.540220:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672720, flags 0x0 00000008:00010000:1.0:1734016141.540223:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016141.540224:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672be0, flags 0x0 00010000:00010000:1.0:1734016141.540238:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540249:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540264:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540271:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540276:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540284:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540289:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540295:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540299:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.540305:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.540369:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335025280:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016141.540419:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016141.540423:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335025152:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016141.546468:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335025280:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016141.546495:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335025152:192.168.201.104@tcp:10:fsx.0 00000080:00200000:1.0:1734016141.637234:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.637245:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.637263:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.637266:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be36f0, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016141.637273:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.637274:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3730, osc lock ffff8800a8672f70, flags 0x400001002 00010000:00010000:1.0:1734016141.637297:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.637306:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.637365:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.637370:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.637408:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335026560:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.637495:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016141.637502:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335026432:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.640664:0:4728:0:(service.c:2100:ptlrpc_server_handle_req_in()) unwrap req x1818247338489856 00000100:00100000:1.0:1734016141.640669:0:4728:0:(service.c:2165:ptlrpc_server_handle_req_in()) got req x1818247338489856 00000100:00100000:1.0:1734016141.640723:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 184 00000100:00100000:1.0:1734016141.640727:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff8800a98b0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7686:x1818247338489856:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016141.640735:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.640741:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016141.640753:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6cbb8[0x4, 9, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016141.640755:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 691702 mtime: 1734016141 atime: 1734016141 ctime: 1734016141 blocks: 48 00000020:00010000:1.0:1734016141.640758:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6c3e8[0x4, 2, [0x280000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016141.640759:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 691702 mtime: 1734016141 atime: 1734016141 ctime: 1734016141 blocks: 48 00000100:00100000:1.0:1734016141.641095:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff8800a98b0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7686:x1818247338489856:12345-192.168.201.104@tcp:106: Request processed in 366us (881us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016141.641101:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 184 00000100:00100000:1.0:1734016141.646138:0:1970:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8800b588c800: committing for last_committed 4294969603 gen 2 00010000:00010000:1.0:1734016141.646172:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646186:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646199:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646207:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646214:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.646228:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 691702, setting rss=691702; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016141.646236:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646241:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646248:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646253:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff880130978b40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.646258:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646267:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016141.646271:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.646276:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee50858 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.646719:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335026432:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.650468:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650488:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650501:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650507:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650512:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.650528:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016141.650558:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650564:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650570:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650577:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097a880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.650582:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650595:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016141.650599:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.650605:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee5085f lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.650632:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335026560:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.666001:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.666009:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.666017:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b7180 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.666023:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.666043:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.666152:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.666190:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.666196:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.666217:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.666224:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7180/0xd17286ca2ee50819 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->20479) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc5599411d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.666242:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.666244:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.666245:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.666252:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016141.666265:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335026816:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016141.674159:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335026816:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016141.757928:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.757935:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.757943:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800b3725440 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016141.757948:0:4738:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.757956:0:4738:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.757963:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.757987:0:4738:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.758023:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.758028:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.758470:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.758477:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b3725440/0xd17286ca2ee50851 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994171 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016141.758500:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.758502:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.758503:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.802419:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.802443:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.802451:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.802458:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee507ef lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080020000 nid: local remote: 0x979ba2bc559940d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.802595:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335028352:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016141.802725:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016141.802733:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335028224:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016141.810185:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335028224:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016141.810199:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335028352:192.168.201.104@tcp:10:fsx.0 00000080:00200000:1.0:1734016141.836641:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 781546 i_blocks 48 00000020:00010000:1.0:1734016141.836652:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8680248 (ffff8800a8711cf8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016141.836656:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8680248 (ffff8800a8711cf8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.836661:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a9971570 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.836665:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a99715b0 (ffff8800a8711908/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016141.836668:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8680248 (ffff8800a8711cf8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.836670:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a9971570 (ffff8800a8711cf8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016141.836675:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a99715b0 (ffff8800a8711cf8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016141.836706:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016141.836731:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016141.836737:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016141.836745:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016141.836763:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.836766:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be39f0, osc lock ffff8800a8673690, flags 0x400001002 00000008:00010000:1.0:1734016141.836771:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016141.836773:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3a30, osc lock ffff8800a8672e40, flags 0x400001002 00010000:00010000:1.0:1734016141.836801:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.836810:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.836872:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff880136af0c00 type 11 mode 6 res 9663677440/7 (0 18446744073709551615) 00010000:00010000:1.0:1734016141.836878:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.836883:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.836924:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335028864:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016141.837009:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016141.837014:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335028992:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.851899:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.851921:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.851935:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.851942:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.851949:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.851964:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016141.851973:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.851980:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.851992:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852000:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff880130978b40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.852004:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852015:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016141.852020:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852026:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130978b40/0xd17286ca2ee508ac lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.852055:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335028992:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.852084:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852091:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852098:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852104:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852110:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.852118:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 781546, setting rss=781546; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016141.852128:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852134:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852141:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852146:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097a880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.852149:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852155:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016141.852158:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.852164:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee508a5 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.852177:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335028864:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.959145:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959162:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959173:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959179:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959184:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016141.959195:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016141.959206:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959211:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959216:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959221:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3840 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.959224:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959233:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016141.959238:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.959244:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee508eb lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016141.959275:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335030272:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016141.964806:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016141.964821:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50889 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559941b0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.964833:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b73c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016141.964841:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50889 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559941b0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016141.964865:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016141.964879:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016141.964928:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50889 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559941b0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.964934:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016141.964971:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50889 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559941b0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.964984:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50889 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559941b0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016141.965004:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.965006:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016141.965007:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016141.965017:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016141.965039:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335030528:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016141.986199:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335030528:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016142.053298:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.053308:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053318:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b6ac0 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016142.053323:0:4738:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053331:0:4738:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053340:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053369:0:4738:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4480/0xd17286ca2ee50786 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993fe9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053423:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053429:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.053472:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053478:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6ac0/0xd17286ca2ee508cf lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994204 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.053509:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.053511:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.053512:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.053523:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016142.053542:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335031296:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.063435:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335031296:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.088924:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335031552:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.097587:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335032192:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.098373:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.098381:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335032320:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.107949:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.107961:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.107976:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.107982:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.107988:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.108000:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4153763, setting rss=4153763; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.108007:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.108013:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.108019:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.108024:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a21c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.108028:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.108036:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.108041:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.108050:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50915 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.108087:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335032192:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.112515:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112541:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112553:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112560:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112567:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.112582:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.112596:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112602:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112613:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112621:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.112626:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112639:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.112645:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.112651:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5091c lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.112684:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335032320:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.137430:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.137437:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137445:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b4fc0 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016142.137450:0:5236:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137457:0:5236:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137468:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137494:0:5236:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4480/0xd17286ca2ee50786 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993fe9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137528:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137534:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.137566:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137572:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee5090e lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994251 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.137595:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.137596:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.137598:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.196454:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.209809:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335034112:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.210516:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335034240:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.226527:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226548:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226567:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226573:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226578:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.226593:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.226600:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226604:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226610:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226615:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.226619:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226628:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.226634:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226644:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5093f lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.226678:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335034240:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.226748:0:1970:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8800aa629800: committing for last_committed 4294969609 gen 1 00010000:00010000:1.0:1734016142.226767:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226774:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226779:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226784:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226788:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.226794:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4591559, setting rss=4591559; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.226821:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226827:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226832:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226836:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1680 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.226839:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226843:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.226846:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.226850:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50938 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.226864:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335034112:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.300239:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800b60e8b40 going to sleep for 11 seconds 00010000:00010000:1.0:1734016142.323872:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.323879:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee508f2 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 270336->278527) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994227 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.323886:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3a80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.323891:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee508f2 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 270336->278527) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994227 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.323913:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016142.323984:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016142.324025:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee508f2 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 270336->278527) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994227 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.324031:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.324065:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee508f2 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 270336->278527) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994227 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.324071:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee508f2 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 270336->278527) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994227 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.324087:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.324088:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.324089:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.324096:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.324113:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335035776:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.336165:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335035776:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016142.336794:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.336800:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a2f40/0xd17286ca2ee50907 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994243 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.336807:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2f40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.336811:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a2f40/0xd17286ca2ee50907 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994243 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.336834:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016142.336847:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016142.336884:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a2f40/0xd17286ca2ee50907 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994243 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.336889:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.336910:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a2f40/0xd17286ca2ee50907 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994243 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.336917:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a2f40/0xd17286ca2ee50907 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994243 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.336942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.336943:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.336944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.350834:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335036160:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.351290:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016142.351297:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335036288:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.352491:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335036032:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.353909:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335036160:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.358565:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335036288:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.392380:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335037184:192.168.201.104@tcp:16:fsx.0 00000100:00100000:1.0:1734016142.404850:0:1969:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8800aa629800: committing for last_committed 4294969615 gen 1 00000100:00100000:1.0:1734016142.404907:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335037184:192.168.201.104@tcp:16:fsx.0 00000100:00100000:1.0:1734016142.406531:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800b60e8cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016142.409922:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335037312:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.409955:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.409965:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.409975:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.409985:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.409996:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.410003:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.410009:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016142.410017:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59c8700 x1818247335037312/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016158 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016142.410028:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016142.410041:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.410069:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016142.410134:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016142.410139:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50985 00000080:00010000:1.0:1734016142.410145:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016142.410148:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50985 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.410150:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.410157:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50985 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559942eb expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.410190:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.410202:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.410224:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.410227:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a86731d0, flags 0x400001002 00000008:00010000:1.0:1734016142.410235:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.410236:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673560, flags 0x400001002 00010000:00010000:1.0:1734016142.410256:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.410263:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.410355:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.410361:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.410399:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335037568:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.410486:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016142.410491:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335037440:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.454414:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454438:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454452:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454459:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454466:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.454481:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5257466, setting rss=5257466; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.454490:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454497:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454509:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454517:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097af40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.454522:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454533:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.454539:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.454545:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee5098c lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.454747:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335037440:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.457010:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457031:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457045:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457063:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457071:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.457086:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.457111:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457119:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457126:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457134:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097ba80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.457139:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457151:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.457156:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.457162:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50993 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.457189:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335037568:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.480844:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335038080:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016142.489642:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335038080:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016142.523231:0:4728:0:(service.c:2100:ptlrpc_server_handle_req_in()) unwrap req x1818247338497152 00000100:00100000:1.0:1734016142.523235:0:4728:0:(service.c:2165:ptlrpc_server_handle_req_in()) got req x1818247338497152 00000100:00100000:1.0:1734016142.523264:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 185 00000100:00100000:1.0:1734016142.523269:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff88012cc30a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7684:x1818247338497152:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016142.523274:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.523280:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016142.523291:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6cbb8[0x4, 3, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016142.523292:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 5257466 mtime: 1734016142 atime: 1734016142 ctime: 1734016142 blocks: 88 00000020:00010000:1.0:1734016142.523295:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6c3e8[0x4, 2, [0x280000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016142.523296:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 5257466 mtime: 1734016142 atime: 1734016142 ctime: 1734016142 blocks: 88 00000100:00100000:1.0:1734016142.523986:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff88012cc30a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7684:x1818247338497152:12345-192.168.201.104@tcp:106: Request processed in 714us (820us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016142.526410:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 185 00000100:00100000:1.0:1734016142.542966:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335039232:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016142.546586:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335039104:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.548755:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335039232:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016142.549738:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335039360:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.556295:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556317:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556332:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556337:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556342:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.556356:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5257466, setting rss=5257466; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.556382:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556388:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556395:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556402:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a18c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.556407:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556417:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.556423:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.556429:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a18c0/0xd17286ca2ee509e0 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.557230:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335039360:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.557253:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016142.557264:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335039488:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.565083:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565104:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565118:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565124:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565131:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.565150:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5257466, setting rss=5257466; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.565973:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565982:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565990:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.565997:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3a80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.566002:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.566013:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.566019:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.566024:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee509e7 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.566108:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335039488:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.567209:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.567226:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335039616:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.574399:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574421:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574435:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574441:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574447:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.574461:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5257466, setting rss=5257466; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.574485:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574490:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574496:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574502:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1d40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.574506:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574521:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.574526:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.574531:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1d40/0xd17286ca2ee509ee lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.575002:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335039616:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.575031:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335039744:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.583494:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583514:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583525:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583531:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583535:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.583548:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5257466, setting rss=5257466; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.583890:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583896:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583902:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583907:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a0fc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.583911:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583921:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.583925:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.583931:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0fc0/0xd17286ca2ee509f5 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.583971:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335039744:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.590269:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.590277:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590284:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800b37258c0 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016142.590288:0:5236:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590295:0:5236:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590303:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590328:0:5236:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4480/0xd17286ca2ee50786 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993fe9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590365:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590369:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.590406:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590412:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800b37258c0/0xd17286ca2ee509a8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994315 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.590437:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.590439:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.590440:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.590448:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.590464:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335040128:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.600751:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335040128:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016142.608042:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.608062:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.608075:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b58c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.608081:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.608111:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016142.608890:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016142.608947:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.608954:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.608983:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.608990:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50977 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559942d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.609012:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.609013:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.609016:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.627087:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335040512:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.635300:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335040384:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.640617:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335040512:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.661420:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335041024:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016142.722383:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.722390:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50a0a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2674688->2678783) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994385 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.722399:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a21c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.722405:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50a0a lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2674688->2678783) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994385 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.722428:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016142.722442:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016142.722483:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50a0a lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2674688->2678783) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994385 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.722489:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.722730:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50a0a lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2674688->2678783) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994385 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.722738:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a21c0/0xd17286ca2ee50a0a lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2674688->2678783) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994385 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.722853:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.722856:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.722858:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.722870:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.722889:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335041920:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.736124:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335042048:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.739552:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335041920:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.749562:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335042048:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016142.756748:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335042432:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.777894:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.777915:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.777928:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.777935:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.777942:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.777958:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4676394, setting rss=4676394; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.777966:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.777971:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.777983:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.778003:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a0fc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.778009:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.778019:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.778024:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.778030:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0fc0/0xd17286ca2ee50a2d lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.778068:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335042432:192.168.201.104@tcp:101:fsx.0 00000080:00200000:1.0:1734016142.778987:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.778992:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.778995:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.779001:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8630 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.779004:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8670 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.779007:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.779009:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8630 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.779012:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8670 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.779023:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.779051:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016142.779057:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.779065:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.779079:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.779081:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016142.779087:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.779088:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016142.779108:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a3b lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.779114:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a3b lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.779178:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50a42 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.779184:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50a42 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.795907:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.795916:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.795919:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.795924:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.795928:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.795931:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.795932:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.796024:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.796038:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.796058:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200000:1.0:1734016142.796112:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016142.796118:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016142.796130:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016142.796144:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2092, 2093] 00000008:00010000:1.0:1734016142.796157:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016142.796160:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea2b0, osc lock ffff8800a86730a0, flags 0x0 00010000:00010000:1.0:1734016142.796177:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.796188:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4374528 4382719) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.796196:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.796201:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.796207:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.796215:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.802824:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774d6c8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.802828:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774d6c8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.802834:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea2b0 (ffff8800a8711878/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016142.802844:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.802857:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016142.802876:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774d6c8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.802878:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea2b0 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.802896:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.802921:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016142.802927:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016142.802942:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016142.802956:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016142.803004:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016142.803010:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016142.803018:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016142.803035:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335043072:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.803121:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3c00 going to sleep for 11 seconds 00000100:00100000:1.0:1734016142.809547:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335043072:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.809574:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.809587:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.809597:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.809611:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.809623:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.809629:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016142.809636:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016142.809647:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59dd500 x1818247335043072/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016158 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016142.809658:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016142.809670:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.809683:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016142.809781:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016142.809787:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50a49 00000080:00010000:1.0:1734016142.809792:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016142.809795:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a49 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.809798:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.809810:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.809845:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.809856:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.809875:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.809878:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86730a0, flags 0x400001002 00000008:00010000:1.0:1734016142.809885:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.809887:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016142.809904:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809916:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809928:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809935:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809951:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809958:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809971:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.809977:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.809996:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.810000:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.810003:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810005:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810009:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810011:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810013:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810016:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.810025:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.810057:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016142.810061:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.810064:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.810069:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810071:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016142.810073:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810075:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016142.810081:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810088:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810095:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810101:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810111:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810117:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810123:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810129:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.810137:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.810140:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.810142:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810144:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810146:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810148:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810150:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810153:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.810157:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.810163:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200008:1.0:1734016142.810178:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016142.810184:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016142.810197:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016142.810204:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.810211:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016142.810218:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016142.810220:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50a49 00000080:00010000:1.0:1734016142.810223:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a49 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.810226:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.810232:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.810240:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.810244:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.810248:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810249:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86730a0, flags 0x400001002 00000008:00010000:1.0:1734016142.810251:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810253:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016142.810258:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810269:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810276:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810282:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810291:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810298:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810304:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810310:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.810318:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.810321:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.810323:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810325:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810328:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810330:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810331:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810334:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.810338:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.810345:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016142.810348:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.810350:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.810354:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810355:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016142.810358:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810359:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016142.810364:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810371:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810379:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810385:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810394:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810400:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810407:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810413:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.810421:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.810423:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.810425:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810427:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810430:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.810431:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810432:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.810435:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.810439:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.810443:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200008:1.0:1734016142.810449:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016142.810453:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016142.810462:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016142.810470:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.810477:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016142.810483:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016142.810486:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50a26 00000080:00010000:1.0:1734016142.810488:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a26 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.810491:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.810495:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.810507:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.810512:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.810518:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810519:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86730a0, flags 0x400001002 00000008:00010000:1.0:1734016142.810522:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.810524:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016142.810540:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50a50 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810546:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50a50 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810606:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50a57 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.810612:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50a57 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.814381:0:4728:0:(service.c:2100:ptlrpc_server_handle_req_in()) unwrap req x1818247338499840 00000100:00100000:1.0:1734016142.814385:0:4728:0:(service.c:2165:ptlrpc_server_handle_req_in()) got req x1818247338499840 00000100:00100000:1.0:1734016142.814419:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 186 00000100:00100000:1.0:1734016142.814424:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff88012cc33b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7686:x1818247338499840:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016142.814431:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.814438:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016142.814451:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b369d3e8[0x4, 1, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016142.814453:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 4676394 mtime: 1734016142 atime: 1734016142 ctime: 1734016142 blocks: 56 00000020:00010000:1.0:1734016142.814455:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b369d4b0[0x4, 2, [0x280000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016142.814456:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 4676394 mtime: 1734016142 atime: 1734016142 ctime: 1734016142 blocks: 56 00000100:00100000:1.0:1734016142.814477:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff88012cc33b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7686:x1818247338499840:12345-192.168.201.104@tcp:106: Request processed in 53us (472us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016142.814482:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 186 00000080:00200000:1.0:1734016142.819563:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.819571:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.819574:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.819579:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.819583:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.819586:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.819588:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.819591:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.819604:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.819627:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016142.819632:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.819640:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.819654:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.819656:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016142.819661:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.819662:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016142.819682:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978d80/0xd17286ca2ee50a5e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.819839:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978d80/0xd17286ca2ee50a5e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.819888:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50a65 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.819894:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50a65 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.827564:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.827573:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.827576:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.827581:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.827584:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.827587:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.827589:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.827593:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.827606:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.827618:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200000:1.0:1734016142.827679:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016142.827684:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016142.827722:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016142.827735:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1047, 1048] 00000008:00010000:1.0:1734016142.827748:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016142.827753:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea2b0, osc lock ffff8800a86730a0, flags 0x0 00010000:00010000:1.0:1734016142.827775:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.827781:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [94208->102399] (req 94208->102399) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.827812:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [94208->102399] (req 94208->102399) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.827835:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335043712:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016142.827912:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3900 going to sleep for 11 seconds 00010000:00010000:1.0:1734016142.831877:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.831885:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.831894:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b5f80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.831899:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.831922:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016142.831936:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016142.831978:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.831984:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.832017:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.832028:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5f80/0xd17286ca2ee509d9 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1515520->1519615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994346 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.832070:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.832071:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.832073:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.836031:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335043712:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.836058:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.836079:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.836090:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.836100:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.836105:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.836131:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.842141:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a774d6c8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.842145:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a774d6c8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.842151:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea2b0 (ffff8800a8711878/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016142.842161:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.842175:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016142.842189:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a774d6c8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.842191:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea2b0 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.842220:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.842245:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016142.842251:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016142.842270:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016142.842286:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.842296:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016142.842306:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016142.842309:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50a26 00000080:00010000:1.0:1734016142.842313:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a26 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.842316:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.842322:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.842342:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.842350:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.842365:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.842367:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3930, osc lock ffff8800a86730a0, flags 0x400001002 00000008:00010000:1.0:1734016142.842372:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.842373:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3970, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016142.842395:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978d80/0xd17286ca2ee50a73 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.842401:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978d80/0xd17286ca2ee50a73 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.842446:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.842452:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.842460:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.842466:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.849541:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.849549:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.849551:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.849557:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3930 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.849560:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3970 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.849563:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.849565:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3930 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.849569:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3970 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.849581:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.849605:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016142.849610:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.849619:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.849632:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.849635:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3930, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016142.849638:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.849640:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3970, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016142.849658:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50a7a lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.849665:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50a7a lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.849747:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.849756:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.849764:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.849770:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.859230:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.859238:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.859241:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.859247:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3930 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.859251:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3970 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.859255:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.859257:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3930 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.859260:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3970 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.859273:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.859286:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200008:1.0:1734016142.859309:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016142.859316:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016142.859333:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016142.859348:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.859357:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016142.859369:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016142.859372:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50a26 00000080:00010000:1.0:1734016142.859378:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a26 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.859381:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.859387:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.859408:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.859416:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.859432:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.859435:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3930, osc lock ffff8800a86730a0, flags 0x400001002 00000008:00010000:1.0:1734016142.859439:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.859440:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3970, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016142.859457:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50a81 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.859463:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50a81 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.859509:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.859516:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.859523:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.859529:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.864437:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.864444:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.864447:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.864452:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3930 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.864456:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3970 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.864459:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.864461:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3930 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.864464:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3970 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.865771:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.865799:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016142.865807:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.865816:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.865833:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.865835:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3930, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016142.865840:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.865842:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3970, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016142.865868:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50a88 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.865877:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50a88 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.865933:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.865942:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.865952:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.865960:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.880074:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 64 00000020:00010000:1.0:1734016142.880083:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_release():233 00000020:00010000:1.0:1734016142.880086:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.880091:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3930 (ffff8800a87113f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.880095:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3970 (ffff8800a8711878/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016142.880099:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8681da8 (ffff8800a8711908/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.880101:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3930 (ffff8800a8711908/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016142.880105:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3970 (ffff8800a8711908/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016142.880117:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016142.880128:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200008:1.0:1734016142.880149:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016142.880156:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016142.880175:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016142.880191:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.880202:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016142.880214:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016142.880217:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50a26 00000080:00010000:1.0:1734016142.880223:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a26 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016142.880226:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016142.880233:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016142.880255:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016142.880264:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016142.880277:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.880280:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3930, osc lock ffff8800a86730a0, flags 0x400001002 00000008:00010000:1.0:1734016142.880284:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016142.880286:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3970, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016142.880305:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.880312:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.880364:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.880372:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.880380:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.880386:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50a6c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 94208->102399) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559943fc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.880401:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.880414:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335044608:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016142.886526:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.886544:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.886558:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.886565:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.886572:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.886587:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4676394, setting rss=4676394; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016142.887205:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.887214:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.887222:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.887228:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097b600 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.887233:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.887247:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016142.887252:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.887258:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b600/0xd17286ca2ee50a8f lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.887306:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335044608:192.168.201.104@tcp:101:fsx.0 00000008:00010000:1.0:1734016142.908002:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e83c0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50aa4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3661824->3665919) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994434 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.908037:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3661824 3665919) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50aa4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3661824->3665919) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994434 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016142.908077:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59ddc00 - object 0x240000400:7 offset 3661824<>3665920 00000100:00100000:1.0:1734016142.909055:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016142.909066:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335045376:192.168.201.104@tcp:3:ptlrpcd_00_00.0 00000008:00010000:1.0:1734016142.913108:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e83c0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50aa4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3661824->3665919) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994434 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.913142:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e83c0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50aa4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3661824->3665919) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994434 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.913166:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335045376:192.168.201.104@tcp:3:ptlrpcd_00_00.0 00000100:00100000:1.0:1734016142.914709:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335045120:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016142.914971:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016142.914985:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335045504:192.168.201.104@tcp:4:fsx.0 00000008:00010000:1.0:1734016142.926993:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8300 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50aa4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3661824->3665919) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994434 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016142.928626:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8300 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50aa4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3661824->3665919) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994434 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016142.928664:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335045504:192.168.201.104@tcp:4:fsx.0 00000100:00100000:1.0:1734016142.933263:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016142.933284:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335045760:192.168.201.104@tcp:16:fsx.0 00000100:00100000:1.0:1734016142.943962:0:1969:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8800b588c800: committing for last_committed 4294968041 gen 2 00000100:00100000:1.0:1734016142.944183:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335045760:192.168.201.104@tcp:16:fsx.0 00010000:00010000:1.0:1734016142.965178:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016142.965187:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.965196:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3a80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016142.965202:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016142.965227:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016142.965244:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016142.965291:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.965298:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016142.965335:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.965342:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50a1f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559943a1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016142.965373:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.965374:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016142.965376:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016142.983351:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335046528:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016143.025080:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016143.025086:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50ac7 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1085439) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994457 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.025094:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b73c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.025099:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50ac7 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1085439) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994457 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.025114:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016143.025199:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016143.025234:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50ac7 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1085439) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994457 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.025245:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016143.025256:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50ac7 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1085439) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994457 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.025262:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50ac7 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1085439) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994457 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.025277:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016143.025278:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016143.025281:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1734016143.092886:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 88 00000020:00010000:1.0:1734016143.092901:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682910 (ffff8800a8711f38/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.092906:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682910 (ffff8800a8711f38/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.092912:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8630 (ffff8800a8711bd8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.092917:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8670 (ffff8800a8711368/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.092921:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682910 (ffff8800a8711f38/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.092923:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8630 (ffff8800a8711f38/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.092930:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8670 (ffff8800a8711f38/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.092945:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.092966:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.092972:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.092979:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.092992:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.092995:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a86737c0, flags 0x400001002 00000008:00010000:1.0:1734016143.093002:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.093003:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a86738f0, flags 0x400001002 00010000:00010000:1.0:1734016143.093029:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b06 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.093048:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b06 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.093118:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50b0d lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.093125:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50b0d lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.193907:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016143.210066:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335050240:192.168.201.104@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1734016143.217726:0:10407:0:(vvp_io.c:899:vvp_io_read_start()) count:1001 iocb pos:6487800 00000100:00100000:1.0:1734016143.217814:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000008:00010000:1.0:1734016143.217827:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be39c0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.217854:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (2289664 2293759) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016143.217885:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59ded80 - object 0x240000400:7 offset 2289664<>2293760 00000100:00100000:1.0:1734016143.217906:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335050496:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016143.228777:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be39c0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.228911:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be39c0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.228935:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335050496:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016143.228952:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335050240:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016143.314504:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1734016143.314511:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50b45 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x979ba2bc559944d5 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314519:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b7a80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.314523:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50b45 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x979ba2bc559944d5 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.314555:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016143.314564:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016143.314592:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50b45 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc559944d5 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314600:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016143.314616:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50b45 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc559944d5 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314625:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee50b45 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc559944d5 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314640:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016143.314643:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314650:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.314653:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.314662:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016143.314731:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016143.314751:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314757:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016143.314763:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314770:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee50b30 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2289664->2297855) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559944c0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.314784:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016143.314785:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016143.314787:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016143.314804:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335051520:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.314850:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016143.314856:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335051392:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.319932:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335051520:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.386631:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335052160:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.405901:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405909:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405918:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405923:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405928:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.405939:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4676394, setting rss=4676394; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016143.405946:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405951:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405956:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405961:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1d40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.405965:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405973:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016143.405977:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.405982:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50b53 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.406005:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335052160:192.168.201.104@tcp:101:fsx.0 00000080:00200000:1.0:1734016143.406395:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 96 00000020:00010000:1.0:1734016143.406400:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.406403:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.406406:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8270 (ffff8800a8711758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.406409:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e82b0 (ffff8800a8711bd8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.406411:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.406413:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8270 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.406416:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e82b0 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.406425:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.406439:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.406444:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.406449:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.406457:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.406459:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be36f0, osc lock ffff8800a8673c80, flags 0x400001002 00000008:00010000:1.0:1734016143.406462:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.406463:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3730, osc lock ffff8800a8673db0, flags 0x400001002 00010000:00010000:1.0:1734016143.406478:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.406488:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.406520:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.406524:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.406544:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335052416:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.406584:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.406588:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335052544:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.419860:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419875:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419884:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419889:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419894:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.419905:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016143.419911:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419916:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419922:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419927:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097af40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.419930:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419937:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016143.419940:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419945:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50b68 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.419974:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335052544:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.419986:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.419997:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420002:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420006:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420011:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.420015:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4676394, setting rss=4676394; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016143.420026:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420057:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420062:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420066:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097bcc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.420069:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420077:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016143.420079:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.420084:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50b61 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.420093:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335052416:192.168.201.104@tcp:101:fsx.0 00000080:00200000:1.0:1734016143.427057:0:10407:0:(rw26.c:1006:ll_write_end()) pos 942976, len 3200, copied 3200 00000080:00200000:1.0:1734016143.427064:0:10407:0:(rw26.c:1044:ll_write_end()) page@ffff8800a74636c0[3 ffff8800b369d468 2 1 ffff88012db7a4e0] 00000080:00200000:1.0:1734016143.427067:0:10407:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a7c0c0 1fffff0000080d 3:0 ffff8800a74636c0 230 lru 00000080:00200000:1.0:1734016143.427072:0:10407:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7463720 230: 1< 1 - - > 2< 942080 0 4096 0x0 0x40009 | ffff8800aa645cc8 ffff8800b6dc4680 > 3< 0 0 > 4< 0 0 8 489881600 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1734016143.427074:0:10407:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a74636c0 00000080:00200000:1.0:1734016143.427075:0:10407:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016143.427079:0:10407:0:(rw26.c:819:ll_write_begin()) Writing 231 of 0 to 2447 bytes 00000080:00200000:1.0:1734016143.433529:0:10407:0:(rw26.c:1006:ll_write_end()) pos 946176, len 2447, copied 2447 00000080:00200000:1.0:1734016143.433537:0:10407:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71de1b0[4 ffff8800b369d468 2 1 ffff88012db7a4e0] 00000080:00200000:1.0:1734016143.433540:0:10407:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0002a5bfc0 1fffff0000080d 3:0 ffff8800a71de1b0 231 lru 00000080:00200000:1.0:1734016143.433546:0:10407:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71de210 231: 1< 1 - - > 2< 946176 0 4096 0x0 0x40009 | ffff8800aa645cc8 ffff8800b6dc4680 > 3< 0 0 > 4< 1 0 8 489881600 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1734016143.433548:0:10407:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71de1b0 00000080:00200000:1.0:1734016143.433548:0:10407:0:(rw26.c:1044:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1734016143.433555:0:10407:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 2, from 896, to 2447 00000008:00010000:1.0:1734016143.433576:0:10407:0:(osc_cache.c:783:osc_extent_find()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.433594:0:10407:0:(osc_cache.c:2323:osc_queue_async_io()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.433610:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afc89248, count 2, dirtied 2 00000080:00200000:1.0:1734016143.433613:0:10407:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 2 pages 5647 bytes, tot: 5647 00000080:00200000:1.0:1734016143.433615:0:10407:0:(vvp_io.c:1405:vvp_io_write_start()) f16b.sanityn: write bytes 0, result: 5647 00000020:00010000:1.0:1734016143.433627:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a7276128 (ffff88012bf4bea8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.433630:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a7276128 (ffff88012bf4bea8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433633:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b3dbbb30 (ffff8800a8711758/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016143.433639:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433649:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016143.433660:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a7276128 (ffff88012bf4bea8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.433662:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b3dbbb30 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.433668:0:10407:0:(vvp_io.c:533:vvp_io_advance()) advancing 5647 bytes 00000080:00200000:1.0:1734016143.433674:0:10407:0:(file.c:1930:ll_file_io_generic()) Range unlock [230, 231] 00000080:00200000:1.0:1734016143.433682:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016143.433685:0:10407:0:(file.c:1964:ll_file_io_generic()) f16b.sanityn: 2 io complete with rc: 0, result: 5647, restart: 0 00000080:00200000:1.0:1734016143.433709:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 96 00000080:00200000:1.0:1734016143.433713:0:10407:0:(file.c:2044:ll_file_io_generic()) iot: 2, result: 5647 00000080:00200008:1.0:1734016143.433728:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016143.433733:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016143.433748:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016143.433759:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.433767:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016143.433777:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016143.433780:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50a49 00000080:00010000:1.0:1734016143.433785:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50a49 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016143.433787:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.433792:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50a49 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559943d9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016143.433805:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.433811:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.433820:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.433822:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673b50, flags 0x400001002 00000008:00010000:1.0:1734016143.433825:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.433826:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673a20, flags 0x400001002 00010000:00010000:1.0:1734016143.433834:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433840:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433847:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 7/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433852:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 6/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433865:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433871:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433876:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433881:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.433888:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 96 00000020:00010000:1.0:1734016143.433890:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.433892:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433893:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a8711758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433895:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4b6c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433897:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.433898:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.433900:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.433904:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.433912:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.433915:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.433917:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.433920:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.433921:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673a20, flags 0x400001002 00000008:00010000:1.0:1734016143.433923:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.433924:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673b50, flags 0x400001002 00010000:00010000:1.0:1734016143.433927:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433932:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433937:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 7/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433942:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 6/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433948:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433952:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433957:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.433962:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b4c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3424255) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559944dc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.433967:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8870698 i_blocks 96 00000020:00010000:1.0:1734016143.433969:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.433971:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433972:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4b6c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433974:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a8711758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.433975:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a480 (ffff88012bf4bea8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.433976:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.433978:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4bea8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.433981:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.433984:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8870698 00000080:00200008:1.0:1734016143.434047:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 8870698, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 6828474, atime 1734016143.000000000, mtime 1734016143.000000000, ctime 1734016143.000000000 00010000:00010000:1.0:1734016143.434065:0:10407:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.434075:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4480/0xd17286ca2ee50786 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993fe9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.434103:0:10407:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.434107:0:10407:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016143.434109:0:10407:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50a26 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc559943af expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1734016143.434139:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800aa629800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016143.434149:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59dd500 x1818247335053312/t0(0) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016143.434172:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335053312:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016143.434235:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.454363:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335053312:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016143.454399:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016143.454405:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016143.454416:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016143.454432:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016143.454441:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016143.454445:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673b50, flags 0x0 00000008:00010000:1.0:1734016143.454448:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016143.454449:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673a20, flags 0x0 00010000:00010000:1.0:1734016143.454474:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.454482:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.454509:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.454529:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335053568:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.456184:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3c00 going to sleep for 11 seconds 00010000:00010000:1.0:1734016143.466811:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016143.466817:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.466824:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3a80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.466829:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.466869:0:4738:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.466879:0:4738:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.466916:0:4738:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (942080 946175) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016143.466977:0:4738:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dfb80 - object 0x280000400:7 offset 942080<>950272 00000100:00100000:1.0:1734016143.467860:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.467874:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335053696:192.168.201.104@tcp:4:fsx.0 00000100:00100000:1.0:1734016143.470900:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335053568:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.470925:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.470935:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.470941:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.477587:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.477633:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50ad5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2658304->2670591) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994465 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.477646:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335053696:192.168.201.104@tcp:4:fsx.0 00010000:00010000:1.0:1734016143.488298:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488308:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488327:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.488337:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4676394, setting rss=4676394; kms=4676394, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488345:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488351:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488376:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488385:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.488407:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.488425:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335053952:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.488489:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3780 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.498601:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335053952:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.498625:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.498635:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.498642:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.499144:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.499151:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.499168:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.499178:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.499187:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.499192:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.499255:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.499265:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335054208:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.499315:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335054336:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.507878:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335054336:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.507895:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335054208:192.168.201.104@tcp:10:fsx.0 00000080:00200000:1.0:1734016143.519340:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016143.519354:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50b7d 00000080:00010000:1.0:1734016143.519361:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016143.519363:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50b7d from it ffff8800a872fcd8 00010000:00010000:1.0:1734016143.519367:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b7d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc5599451b expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.519377:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b7d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599451b expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016143.519411:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.519417:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.519447:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.519449:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37b0, osc lock ffff8800a8673a20, flags 0x400001002 00000008:00010000:1.0:1734016143.519460:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.519461:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37f0, osc lock ffff8800a8673b50, flags 0x400001002 00010000:00010000:1.0:1734016143.519477:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50b84 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.519483:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50b84 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.519543:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.519557:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335054592:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.522113:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50b8b lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.522127:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50b8b lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.534680:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 6828474 i_blocks 104 00000020:00010000:1.0:1734016143.534686:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.534688:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.534721:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37b0 (ffff88012bf4b2d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.534725:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37f0 (ffff88012bf4bea8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.534727:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.534729:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37b0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.534732:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37f0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.534741:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.534756:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.534761:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.534767:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.534778:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.534779:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37b0, osc lock ffff8800a8673b50, flags 0x400001002 00000008:00010000:1.0:1734016143.534782:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.534783:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37f0, osc lock ffff8800a8673a20, flags 0x400001002 00010000:00010000:1.0:1734016143.534798:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50b92 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.534804:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097af40/0xd17286ca2ee50b92 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.534845:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee50b99 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.534849:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee50b99 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.545569:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 6828474 i_blocks 104 00000020:00010000:1.0:1734016143.545574:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.545577:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.545581:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37b0 (ffff88012bf4bea8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.545584:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37f0 (ffff88012bf4b2d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.545586:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.545588:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37b0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.545592:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37f0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.545602:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.545610:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 6828474 00000080:00200008:1.0:1734016143.545623:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016143.545629:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016143.545643:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016143.545652:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b3e08000 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016143.545686:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016143.545727:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016143.545736:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016143.545751:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335055104:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.545809:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3780 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.549401:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335055104:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.549419:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.549428:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.549436:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.549443:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.549455:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.549461:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.549466:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016143.549474:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59df100 x1818247335055104/t0(0) o101->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016159 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016143.549483:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016143.549492:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.549501:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b3e08000 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016143.549542:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016143.549548:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50ba0 00000080:00010000:1.0:1734016143.549554:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016143.549557:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50ba0 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016143.549559:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.549565:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff88013097ad00/0xd17286ca2ee50ba0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994545 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016143.549591:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.549598:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.549609:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.549611:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37b0, osc lock ffff8800a8673a20, flags 0x400001002 00000008:00010000:1.0:1734016143.549615:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.549616:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37f0, osc lock ffff8800a8673b50, flags 0x400001002 00010000:00010000:1.0:1734016143.549627:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549635:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549643:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549649:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549659:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549664:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549670:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549680:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.549761:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 6828474 i_blocks 104 00000020:00010000:1.0:1734016143.549765:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.549767:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.549770:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37b0 (ffff88012bf4b2d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.549772:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37f0 (ffff88012bf4bea8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.549775:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.549776:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37b0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.549779:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37f0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.549785:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.549798:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.549801:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.549804:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.549808:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.549809:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37b0, osc lock ffff8800a8673b50, flags 0x400001002 00000008:00010000:1.0:1734016143.549811:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.549813:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37f0, osc lock ffff8800a8673a20, flags 0x400001002 00010000:00010000:1.0:1734016143.549818:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549824:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549830:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549836:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50b6f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994506 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549845:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549850:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549857:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.549862:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50b76 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599450d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.549869:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 6828474 i_blocks 104 00000020:00010000:1.0:1734016143.549871:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.549872:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.549874:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37b0 (ffff88012bf4bea8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.549876:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be37f0 (ffff88012bf4b2d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.549878:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.549879:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37b0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.549881:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be37f0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.549884:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.549889:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 6828474 00000080:00200008:1.0:1734016143.550203:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 6828474, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 8158164, atime 1734016143.000000000, mtime 1734016143.000000000, ctime 1734016143.000000000 00010000:00010000:1.0:1734016143.550228:0:10407:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b7d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc5599451b expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.550239:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.550266:0:10407:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b7d lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc5599451b expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.550271:0:10407:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016143.550273:0:10407:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5b00/0xd17286ca2ee50b7d lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc5599451b expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1734016143.550306:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016143.550313:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59df100 x1818247335055360/t0(0) o36->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016143.550333:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335055360:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016143.550384:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3780 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.557556:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335055360:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016143.557595:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016143.557601:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016143.557614:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016143.557632:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016143.557643:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016143.557646:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37b0, osc lock ffff8800a8673a20, flags 0x0 00000008:00010000:1.0:1734016143.557648:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016143.557649:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be37f0, osc lock ffff8800a8673b50, flags 0x0 00010000:00010000:1.0:1734016143.557671:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.557678:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.557732:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.557752:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335055616:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.557788:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3e40 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.561729:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335055744:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.568256:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335055744:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.568512:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335056000:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.571536:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335056256:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.571856:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016143.571863:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335056128:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.572868:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335056000:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.574343:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335056128:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.574355:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335056256:192.168.201.104@tcp:10:fsx.0 00000008:00010000:1.0:1734016143.575102:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8cc0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.575130:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3956736 3960831) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016143.575159:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59ddf80 - object 0x240000400:7 offset 3956736<>3960832 00000100:00100000:1.0:1734016143.575173:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.575182:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335056512:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016143.580119:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8cc0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.580142:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8cc0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.580162:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335056512:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016143.581175:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:1.0:1734016143.581186:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8b40 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.581205:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3960832 3964927) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016143.581229:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59ddf80 - object 0x240000400:7 offset 3960832<>3964928 00000100:00100000:1.0:1734016143.581247:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335056768:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016143.585137:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8b40 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.585175:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8b40 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.585195:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335056768:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016143.585674:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335056896:192.168.201.104@tcp:4:fsx.0 00000008:00010000:1.0:1734016143.594068:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8c00 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.594096:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8c00 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.594113:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335056896:192.168.201.104@tcp:4:fsx.0 00000100:00100000:1.0:1734016143.597928:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.597945:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335057152:192.168.201.104@tcp:16:fsx.0 00000100:00100000:1.0:1734016143.605153:0:1970:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8800b588c800: committing for last_committed 4294968045 gen 2 00000100:00100000:1.0:1734016143.605225:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335057152:192.168.201.104@tcp:16:fsx.0 00000080:00200000:1.0:1734016143.611344:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 120 00000020:00010000:1.0:1734016143.611349:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.611351:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.611354:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8c30 (ffff88012bf4b2d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.611357:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8c70 (ffff88012bf4bea8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.611359:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.611361:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8c30 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.611365:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8c70 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.611373:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.611386:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.611391:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.611394:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.611402:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.611403:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be36f0, osc lock ffff8800a8673db0, flags 0x400001002 00000008:00010000:1.0:1734016143.611412:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.611413:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3730, osc lock ffff8800a8673c80, flags 0x400001002 00010000:00010000:1.0:1734016143.611420:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611427:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611434:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611440:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611449:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611453:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611464:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.611469:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.611477:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 120 00000020:00010000:1.0:1734016143.611479:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.611480:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.611482:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be36f0 (ffff88012bf4bea8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.611484:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3730 (ffff88012bf4b2d8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.611485:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682da0 (ffff8800a8711bd8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.611487:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be36f0 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.611489:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3730 (ffff8800a8711bd8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.611494:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.611498:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8158164 00000080:00200008:1.0:1734016143.611509:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016143.611514:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016143.611525:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016143.611533:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b3e08000 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016143.611561:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016143.611566:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016143.611576:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016143.611592:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335057408:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.611658:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be36c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.617486:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335057408:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.617508:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.617517:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.617526:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.617535:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.617544:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.617550:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016143.617556:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016143.617565:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59ddf80 x1818247335057408/t0(0) o101->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016159 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016143.617576:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016143.617586:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.617597:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b3e08000 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016143.617640:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016143.617645:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50bbc 00000080:00010000:1.0:1734016143.617650:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016143.617653:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50bbc from it ffff8800a872fcd8 00010000:00010000:1.0:1734016143.617656:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.617662:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016143.617722:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.617731:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.617744:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.617747:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be36f0, osc lock ffff8800a8673c80, flags 0x400001002 00000008:00010000:1.0:1734016143.617752:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.617753:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3730, osc lock ffff8800a8673db0, flags 0x400001002 00010000:00010000:1.0:1734016143.617770:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.617777:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.617836:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.617843:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.617874:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335057664:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.617938:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016143.617944:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335057536:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.625196:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625211:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625222:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625227:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625233:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.625245:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016143.625252:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625258:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625264:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625270:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097a640 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.625273:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625302:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016143.625307:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.625314:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bc3 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.626004:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335057536:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.626020:0:1969:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8800aa629800: committing for last_committed 4294968043 gen 2 00010000:00010000:1.0:1734016143.626043:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626051:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626062:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626067:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626073:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.626080:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 3963860, setting rss=3963860; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016143.626093:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626098:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626104:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626108:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097aac0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.626111:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626117:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016143.626120:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.626129:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097aac0/0xd17286ca2ee50bca lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.626141:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335057664:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.627326:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335057920:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.634429:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634438:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634444:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634449:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634455:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.634462:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 3963860, setting rss=3963860; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016143.634470:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634479:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634484:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634488:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1d40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016143.634490:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634496:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016143.634500:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.634504:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1d40/0xd17286ca2ee50bd8 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.634519:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335057920:192.168.201.104@tcp:101:fsx.0 00000008:00010000:1.0:1734016143.635373:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8a80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.635397:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1085440 1089535) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016143.635422:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59df480 - object 0x240000400:7 offset 1085440<>1089536 00000100:00100000:1.0:1734016143.635585:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.635595:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335058176:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016143.640514:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8a80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.640599:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8a80 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.640617:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335058176:192.168.201.104@tcp:3:fsx.0 00000080:00200008:1.0:1734016143.655985:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016143.655993:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016143.656010:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016143.656022:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50bb5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994568 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.656043:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50bb5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994568 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016143.656054:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016143.656058:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50bb5 00000080:00010000:1.0:1734016143.656062:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50bb5 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016143.656065:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50bb5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994568 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.656071:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50bb5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994568 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016143.656091:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.656100:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.656133:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.656136:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86738f0, flags 0x400001002 00000008:00010000:1.0:1734016143.656150:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.656151:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86737c0, flags 0x400001002 00010000:00010000:1.0:1734016143.656162:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656171:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656180:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656185:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656197:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656203:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656209:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656219:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.656233:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 120 00000020:00010000:1.0:1734016143.656235:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.656237:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.656240:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff88012bf4b488/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.656242:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff88012bf4b368/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.656244:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.656246:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff88012bf4b6c8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.656249:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff88012bf4b6c8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.656255:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.656264:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.656268:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.656271:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.656275:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.656277:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86737c0, flags 0x400001002 00000008:00010000:1.0:1734016143.656280:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.656281:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86738f0, flags 0x400001002 00010000:00010000:1.0:1734016143.656286:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656293:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656300:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656306:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656317:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656323:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656330:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656336:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50bae lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599455a expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.656345:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 120 00000020:00010000:1.0:1734016143.656347:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.656349:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.656352:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff88012bf4b368/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.656354:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff88012bf4b488/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.656356:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.656357:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff88012bf4b6c8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.656359:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff88012bf4b6c8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.656364:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.656369:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8158164 00000080:00200008:1.0:1734016143.656382:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016143.656386:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016143.656395:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016143.656403:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.656411:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016143.656417:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016143.656419:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50bbc 00000080:00010000:1.0:1734016143.656422:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50bbc from it ffff8800a872fcd8 00010000:00010000:1.0:1734016143.656425:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016143.656430:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50bbc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994576 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016143.656442:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.656446:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.656451:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.656453:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c30, osc lock ffff8800a86738f0, flags 0x400001002 00000008:00010000:1.0:1734016143.656456:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.656457:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3c70, osc lock ffff8800a86737c0, flags 0x400001002 00010000:00010000:1.0:1734016143.656472:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bdf lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656479:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50bdf lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656529:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50be6 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.656535:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50be6 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.668606:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 120 00000020:00010000:1.0:1734016143.668613:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.668616:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.668621:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c30 (ffff88012bf4b488/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.668625:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3c70 (ffff88012bf4b368/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.668627:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db79ff0 (ffff88012bf4b6c8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.668629:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c30 (ffff88012bf4b6c8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.668633:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3c70 (ffff88012bf4b6c8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.668642:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016143.668659:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016143.668662:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016143.668671:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016143.668681:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1623, 1623] 00000008:00010000:1.0:1734016143.668728:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016143.668731:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a86737c0, flags 0x0 00010000:00010000:1.0:1734016143.668747:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.668755:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2453504->2457599] (req 2453504->2457599) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.668780:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2453504->2457599] (req 2453504->2457599) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.668801:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335059072:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.669606:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3a80 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.677645:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335059072:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.677670:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2453504->2457599] (req 2453504->2457599) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.677679:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2453504->2457599] (req 2453504->2457599) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.677686:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2453504->2457599] (req 2453504->2457599) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.678537:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.678544:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.678562:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.678572:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 3963860, setting rss=3963860; kms=3963860, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.678581:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.678587:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.678608:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 120 00000100:00100000:1.0:1734016143.678664:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.678679:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335059328:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016143.683528:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335059328:192.168.201.104@tcp:22:fsx.0 00000080:00200008:1.0:1734016143.693426:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016143.693434:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.693442:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.693477:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.693479:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a86738f0, flags 0x400001002 00000008:00010000:1.0:1734016143.693492:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.693494:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8673300, flags 0x400001002 00010000:00010000:1.0:1734016143.693512:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50bfb lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.693520:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50bfb lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.693575:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.693584:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.693597:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.693603:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50bed lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2453504->2457599) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945a7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.710105:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 128 00000020:00010000:1.0:1734016143.710112:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a238 (ffff88012bf4b488/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.710115:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a238 (ffff88012bf4b488/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.710119:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff88012bf4b3f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.710122:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff88012bf4bcf8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.710124:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a238 (ffff88012bf4b488/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.710126:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b488/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.710128:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b488/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.710138:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016143.710146:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 8158164 00000080:00a00000:1.0:1734016143.710220:0:10407:0:(llite_mmap.c:611:ll_file_mmap()) VFS_Op: fid=[0x200000401:0xc:0x0] vma=ffff8800b6f451b0 start=0x7f7d584a7000 end=0x7f7d584a9000 vm_flags=0xfb 00000080:00200000:1.0:1734016143.710230:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016143.710237:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016143.710247:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.710249:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016143.710253:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016143.710254:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a86738f0, flags 0x400001002 00010000:00010000:1.0:1734016143.710266:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.710280:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.710288:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.710294:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097b600/0xd17286ca2ee50ba7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994553 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.710314:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c02 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.710320:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c02 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.728301:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 8158164 i_blocks 128 00000020:00010000:1.0:1734016143.728307:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a238 (ffff88012bf4b488/0) at cl_lock_release():233 00000020:00010000:1.0:1734016143.728309:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a238 (ffff88012bf4b488/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.728314:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff88012bf4bcf8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.728317:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff88012bf4b3f8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016143.728319:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a238 (ffff88012bf4b488/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.728320:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b488/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016143.728323:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b488/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016143.728333:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016143.728376:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 1635 00000080:00200000:1.0:1734016143.728383:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016143.728394:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016143.728404:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1635, 1635] 00000008:00010000:1.0:1734016143.728414:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016143.728417:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea2b0, osc lock ffff8800a86738f0, flags 0x0 00010000:00010000:1.0:1734016143.728434:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.728440:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2502656->2506751] (req 2502656->2506751) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.728467:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2502656->2506751] (req 2502656->2506751) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.728489:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335059840:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016143.728854:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be36c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.753063:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335059968:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.765237:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335059840:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.765265:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2502656->2506751] (req 2502656->2506751) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.765276:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2502656->2506751] (req 2502656->2506751) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.765283:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [2502656->2506751] (req 2502656->2506751) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.770373:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.770389:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.770410:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.770422:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 3963860, setting rss=3963860; kms=3963860, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.770432:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.770438:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.771427:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000008:00010000:1.0:1734016143.771440:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be36c0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.771470:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (2502656 2506751) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016143.771503:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dd880 - object 0x240000400:7 offset 2502656<>2506752 00000100:00100000:1.0:1734016143.771525:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335060224:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016143.779953:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be36c0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.779991:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be36c0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee50c09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2502656->2506751) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945c3 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016143.780008:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335060224:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016143.783360:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335059968:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.829934:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335061120:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.838780:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335061120:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016143.852399:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335061248:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.869044:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335061248:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016143.883152:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800b60e8a80 going to sleep for 11 seconds 00000100:00100000:1.0:1734016143.903943:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8800aa629800: committing for last_committed 4294968046 gen 2 00000100:00100000:1.0:1734016143.903972:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335061632:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016143.903994:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.904014:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016143.904036:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 850483, setting rss=850483; kms=850483, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.904052:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016143.904058:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016143.904081:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5044787 i_blocks 96 00000100:00100000:1.0:1734016143.904139:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016143.904153:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335061888:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016143.911396:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335061888:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016143.976284:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335062400:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016144.033252:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.033270:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.033332:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:1.0:1734016144.033349:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be36c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.033386:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3420160 3424255) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016144.033428:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dd880 - object 0x280000400:7 offset 3420160<>3428352 00000100:00100000:1.0:1734016144.033452:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335063552:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016144.034169:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335063296:192.168.201.104@tcp:103:ldlm_bl.0 00000008:00010000:1.0:1734016144.037969:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be36c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.038008:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be36c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.038041:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335063552:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016144.057618:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335063808:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.067787:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067815:0:1970:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067833:0:1970:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067845:0:1970:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.067867:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067892:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067899:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067905:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.067911:0:1970:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50c5d lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x979ba2bc5599462c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.067935:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335063808:192.168.201.104@tcp:101:fsx.0 00000080:00200008:1.0:1734016144.069848:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 6635232, type: SEEK_SET 00000080:00200008:1.0:1734016144.069863:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 6635232, type: SEEK_SET, rc = 6635232 00000080:00200008:1.0:1734016144.069875:0:10407:0:(file.c:2466:do_file_write_iter()) START file f16b.sanityn:[0x200000401:0xc:0x0], ppos: 6635232, count: 4206 00000080:00200000:1.0:1734016144.069886:0:10407:0:(file.c:1826:ll_file_io_generic()) f16b.sanityn: write ppos: 6635232, bytes: 4206 00000080:00200000:1.0:1734016144.069892:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000020:00200000:1.0:1734016144.069898:0:10407:0:(cl_io.c:195:cl_io_rw_init()) header@ffff8800b3d6cbb8[0x4, 1, [0x200000401:0xc:0x0] hash] 00000020:00200000:1.0:1734016144.069900:0:10407:0:(cl_io.c:195:cl_io_rw_init()) io range: 2 [6635232, 6639438) 0 0 00000080:00200000:1.0:1734016144.069905:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016144.069919:0:10407:0:(file.c:1893:ll_file_io_generic()) Range lock [1619, 1620] 00020000:00200000:1.0:1734016144.069928:0:10407:0:(lov_io.c:1021:lov_io_rw_iter_init()) stripe: 1 chunk: [6635232, 6639438) 6639438, 4206 00020000:00200000:1.0:1734016144.069932:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016144.069987:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1619, 1620] 00000008:00010000:1.0:1734016144.070000:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016144.070004:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a8673300, flags 0x0 00010000:00010000:1.0:1734016144.070030:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070044:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (2437120 2445311) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070053:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070061:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070066:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.070074:0:10407:0:(vvp_io.c:1326:vvp_io_write_start()) f16b.sanityn: write [6635232, 6639438) 00000080:00200000:1.0:1734016144.070107:0:10407:0:(rw26.c:819:ll_write_begin()) Writing 1619 of 3808 to 288 bytes 00000080:00200000:1.0:1734016144.070153:0:10407:0:(rw26.c:1006:ll_write_end()) pos 6635232, len 288, copied 288 00000080:00200000:1.0:1734016144.070158:0:10407:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71de288[3 ffff8800b3d6cc38 2 1 ffff88012db7b720] 00000080:00200000:1.0:1734016144.070161:0:10407:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00029bbcc0 1fffff00000805 3:0 ffff8800a71de288 1619 lru 00000080:00200000:1.0:1734016144.070167:0:10407:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71de2e8 595: 1< 0 - - > 2< 2437120 0 0 0x0 0x0 | ffff8800a9984758 ffff8800a85d84e0 > 3< 0 0 > 4< 0 0 8 12656640 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1734016144.070168:0:10407:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71de288 00000080:00200000:1.0:1734016144.070169:0:10407:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016144.070170:0:10407:0:(rw26.c:819:ll_write_begin()) Writing 1620 of 0 to 3918 bytes 00000080:00200000:1.0:1734016144.070182:0:10407:0:(rw26.c:1006:ll_write_end()) pos 6635520, len 3918, copied 3918 00000080:00200000:1.0:1734016144.070185:0:10407:0:(rw26.c:1044:ll_write_end()) page@ffff8800a71de360[3 ffff8800b3d6cc38 2 1 ffff88012db7b720] 00000080:00200000:1.0:1734016144.070187:0:10407:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00029a6400 1fffff00000805 3:0 ffff8800a71de360 1620 lru 00000080:00200000:1.0:1734016144.070191:0:10407:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a71de3c0 596: 1< 0 - - > 2< 2441216 0 0 0x0 0x0 | ffff8800a9984758 ffff8800a85d84e0 > 3< 0 0 > 4< 0 0 8 12656640 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1734016144.070192:0:10407:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a71de360 00000080:00200000:1.0:1734016144.070193:0:10407:0:(rw26.c:1044:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1734016144.070197:0:10407:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 2, from 3808, to 3918 00000008:00010000:1.0:1734016144.070215:0:10407:0:(osc_cache.c:783:osc_extent_find()) ### extent: ffff880136be3c00 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.070231:0:10407:0:(osc_cache.c:2323:osc_queue_async_io()) ### extent: ffff880136be3c00 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.070240:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 2, dirtied 2 00000080:00200000:1.0:1734016144.070244:0:10407:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 2 pages 4206 bytes, tot: 4206 00000080:00200000:1.0:1734016144.070247:0:10407:0:(vvp_io.c:1405:vvp_io_write_start()) f16b.sanityn: write bytes 0, result: 4206 00000020:00010000:1.0:1734016144.070258:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72763f8 (ffff88012bf4bcf8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.070260:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72763f8 (ffff88012bf4bcf8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.070262:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea230 (ffff88012bf4b248/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016144.070265:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070271:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50c17 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 176128->188415) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559945df expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016144.070281:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72763f8 (ffff88012bf4bcf8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.070282:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea230 (ffff88012bf4bcf8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.070286:0:10407:0:(vvp_io.c:533:vvp_io_advance()) advancing 4206 bytes 00000080:00200000:1.0:1734016144.070290:0:10407:0:(file.c:1930:ll_file_io_generic()) Range unlock [1619, 1620] 00000080:00200000:1.0:1734016144.070295:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016144.070298:0:10407:0:(file.c:1964:ll_file_io_generic()) f16b.sanityn: 2 io complete with rc: 0, result: 4206, restart: 0 00000080:00200000:1.0:1734016144.070305:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 6639438 i_blocks 96 00000080:00200000:1.0:1734016144.070309:0:10407:0:(file.c:2044:ll_file_io_generic()) iot: 2, result: 4206 00000080:00200008:1.0:1734016144.070324:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016144.070330:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016144.070349:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016144.070361:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5d40/0xd17286ca2ee50c10 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559945d8 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016144.070368:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5d40/0xd17286ca2ee50c10 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559945d8 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016144.070379:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016144.070382:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50c10 00000080:00010000:1.0:1734016144.070386:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50c10 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016144.070389:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5d40/0xd17286ca2ee50c10 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559945d8 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016144.070394:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b5d40/0xd17286ca2ee50c10 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559945d8 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016144.070410:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016144.070416:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016144.070423:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.070424:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3030, osc lock ffff8800a8673300, flags 0x400001002 00000008:00010000:1.0:1734016144.070441:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.070443:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3070, osc lock ffff8800a86730a0, flags 0x400001002 00010000:00010000:1.0:1734016144.070451:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070458:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070467:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070474:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b4fc0/0xd17286ca2ee50c4f lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3420160->3428351) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994617 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070495:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.070501:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.070563:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016144.070582:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335063936:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.078511:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078537:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078550:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078558:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078569:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.078586:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 2445134, setting rss=2445134; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016144.078620:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078625:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078632:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078640:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff88013097a880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016144.078645:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078707:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016144.078714:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.078721:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50c64 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.078765:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335063936:192.168.201.104@tcp:101:fsx.0 00000008:00010000:1.0:1734016144.220745:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8cc0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b73c0/0xd17286ca2ee50c80 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1368064->1372159) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599464f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.220783:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1368064 1372159) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b73c0/0xd17286ca2ee50c80 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1368064->1372159) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599464f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016144.220821:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dd880 - object 0x240000400:7 offset 1368064<>1372160 00000100:00100000:1.0:1734016144.254732:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335066112:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016144.254782:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016144.254789:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016144.254807:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016144.254862:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016144.254874:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016144.254878:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8673560, flags 0x0 00000008:00010000:1.0:1734016144.254881:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016144.254882:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a86731d0, flags 0x0 00010000:00010000:1.0:1734016144.254899:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff880136af2000 type 11 mode 2 res 10737419264/7 (0 18446744073709551615) 00010000:00010000:1.0:1734016144.254913:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.254923:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.254958:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.254979:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335066368:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016144.255047:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3e40 going to sleep for 11 seconds 00000100:00100000:1.0:1734016144.266950:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335066368:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.266984:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.266996:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.267005:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281483:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281494:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281520:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.281536:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281549:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281558:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281586:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff880136af0c00 type 11 mode 2 res 9663677440/7 (0 18446744073709551615) 00010000:00010000:1.0:1734016144.281596:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281602:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.281628:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.281649:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335066752:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016144.281720:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3e40 going to sleep for 11 seconds 00000100:00100000:1.0:1734016144.301334:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335066752:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.301360:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.301372:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.301379:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.306731:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.306744:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.306765:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.306777:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 2445134, setting rss=2445134; kms=2445134, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.306788:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.306794:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.319785:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 4539372 i_blocks 96 00000020:00010000:1.0:1734016144.319794:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a7276218 (ffff88012bf4b1b8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.319798:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a7276218 (ffff88012bf4b1b8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.319801:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff88012bf4b518/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016144.319812:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.319824:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016144.319844:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff88012bf4ba28/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016144.319848:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.319855:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097b840/0xd17286ca2ee50c95 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599466b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016144.319863:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a7276218 (ffff88012bf4b1b8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.319864:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.319868:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.319889:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016144.319909:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 4539372, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 4539372, atime 1734016144.000000000, mtime 1734016144.240000000, ctime 1734016144.240000000, rc 0 00000080:00200008:1.0:1734016144.319936:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016144.319942:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016144.319961:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016144.319977:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016144.320034:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016144.320040:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016144.320050:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016144.320070:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335067392:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016144.322283:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3a80 going to sleep for 11 seconds 00000100:00100000:1.0:1734016144.327627:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335067392:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.327655:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016144.327665:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016144.327675:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016144.327683:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016144.327769:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016144.327777:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016144.327782:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016144.327792:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59de300 x1818247335067392/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016160 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016144.327803:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097bcc0/0xd17286ca2ee50c9c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994679 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1734016144.345295:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335068416:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016144.349007:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335068416:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016144.349050:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335068288:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016144.360053:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:1.0:1734016144.360072:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8780 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.360124:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (2170880 2174975) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016144.360297:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59de300 - object 0x280000400:7 offset 0<>2191360 00000100:00100000:1.0:1734016144.360325:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335068800:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016144.421386:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8780 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.430500:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8780 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.430553:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335068800:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016144.581411:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335070848:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.596033:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596041:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596051:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596056:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596062:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.596070:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 2187295, setting rss=2187295; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016144.596078:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596084:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596089:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596093:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a33c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016144.596095:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596103:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016144.596105:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.596114:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a33c0/0xd17286ca2ee50cd4 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.596144:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335070848:192.168.201.104@tcp:101:fsx.0 00000080:00200000:1.0:1734016144.606481:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 7881220 i_blocks 120 00000020:00010000:1.0:1734016144.606488:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.606491:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606496:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8330 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606499:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e8370 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606501:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.606502:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8330 (ffff8800a8711368/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.606507:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e8370 (ffff8800a8711368/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.606518:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016144.606530:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 7881220 00000080:00200008:1.0:1734016144.606550:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016144.606556:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016144.606574:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016144.606590:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016144.606600:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016144.606612:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016144.606615:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50ca3 00000080:00010000:1.0:1734016144.606621:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50ca3 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016144.606623:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016144.606629:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016144.606648:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016144.606657:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016144.606672:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.606674:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a86731d0, flags 0x400001002 00000008:00010000:1.0:1734016144.606679:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.606681:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8673560, flags 0x400001002 00010000:00010000:1.0:1734016144.606719:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606727:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606734:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606740:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606752:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606757:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606762:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606767:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.606777:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 7881220 i_blocks 120 00000020:00010000:1.0:1734016144.606780:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.606784:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606786:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606788:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606790:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.606791:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff8800a8711368/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.606794:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff8800a8711368/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.606799:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016144.606811:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016144.606814:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016144.606817:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016144.606821:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.606822:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8673560, flags 0x400001002 00000008:00010000:1.0:1734016144.606825:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.606825:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a86731d0, flags 0x400001002 00010000:00010000:1.0:1734016144.606830:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606837:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606843:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606848:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606857:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606862:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606867:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606872:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b7600/0xd17286ca2ee50cc6 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 868352->876543) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559946b1 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.606878:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 7881220 i_blocks 120 00000020:00010000:1.0:1734016144.606880:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.606882:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606884:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606886:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.606888:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.606889:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff8800a8711368/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.606892:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff8800a8711368/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.606896:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016144.606900:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 7881220 00000080:00200008:1.0:1734016144.606904:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016144.606907:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016144.606916:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016144.606925:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016144.606930:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016144.606935:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016144.606937:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50ccd 00000080:00010000:1.0:1734016144.606941:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50ccd from it ffff8800a872fcd8 00010000:00010000:1.0:1734016144.606943:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016144.606948:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016144.606959:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016144.606962:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016144.606967:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.606968:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a86731d0, flags 0x400001002 00000008:00010000:1.0:1734016144.606971:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.606972:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8673560, flags 0x400001002 00010000:00010000:1.0:1734016144.606985:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50cf0 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.606995:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50cf0 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.607063:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50cf7 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.607068:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50cf7 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.615866:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 7881220 i_blocks 120 00000020:00010000:1.0:1734016144.615872:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.615875:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.615880:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.615884:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.615886:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.615888:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff8800a8711368/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.615891:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff8800a8711368/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.615903:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016144.615925:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016144.615931:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016144.615940:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016144.615953:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.615955:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8673560, flags 0x400001002 00000008:00010000:1.0:1734016144.615960:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.615961:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a86731d0, flags 0x400001002 00010000:00010000:1.0:1734016144.615986:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50cfe lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.615997:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50cfe lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.616063:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee50d05 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.616069:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee50d05 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.620476:0:4728:0:(service.c:171:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1734016144.620487:0:4728:0:(service.c:2100:ptlrpc_server_handle_req_in()) unwrap req x1818247338514432 00000100:00100000:1.0:1734016144.620490:0:4728:0:(service.c:2165:ptlrpc_server_handle_req_in()) got req x1818247338514432 00000100:00100000:1.0:1734016144.620524:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 188 00000100:00100000:1.0:1734016144.620530:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff8800a70d0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7685:x1818247338514432:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016144.620536:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.620543:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee50c8e lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994664 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016144.620556:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b369d3e8[0x4, 536, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016144.620557:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 2187295 mtime: 1734016144 atime: 1734016144 ctime: 1734016144 blocks: 72 00000020:00010000:1.0:1734016144.620560:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b369d4b0[0x4, 2, [0x280000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016144.620561:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 2187295 mtime: 1734016144 atime: 1734016144 ctime: 1734016144 blocks: 72 00000100:00100000:1.0:1734016144.620592:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff8800a70d0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7685:x1818247338514432:12345-192.168.201.104@tcp:106: Request processed in 62us (1124us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016144.620598:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 188 00000080:00200000:1.0:1734016144.624327:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 7881220 i_blocks 120 00000020:00010000:1.0:1734016144.624334:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.624337:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.624342:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.624345:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.624347:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.624349:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff8800a8711368/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.624353:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff8800a8711368/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.624364:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016144.624375:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 7881220 00000080:00a00000:1.0:1734016144.624446:0:10407:0:(llite_mmap.c:611:ll_file_mmap()) VFS_Op: fid=[0x200000401:0xc:0x0] vma=ffff8800b6f450d8 start=0x7f7d584a7000 end=0x7f7d584a9000 vm_flags=0xfb 00000080:00200000:1.0:1734016144.624457:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016144.624466:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016144.624479:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.624482:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a86731d0, flags 0x400001002 00000008:00010000:1.0:1734016144.624486:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016144.624488:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8673560, flags 0x400001002 00010000:00010000:1.0:1734016144.624510:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee50d0c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.624517:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee50d0c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.624564:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50d13 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.624569:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50d13 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016144.632589:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 7881220 i_blocks 120 00000020:00010000:1.0:1734016144.632596:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_release():233 00000020:00010000:1.0:1734016144.632599:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.632604:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a87116c8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.632607:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a8711008/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016144.632610:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a8682238 (ffff8800a8711368/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.632612:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff8800a8711368/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016144.632616:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff8800a8711368/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016144.632629:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016144.633110:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 1346 00000080:00200000:1.0:1734016144.633120:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016144.633131:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016144.633142:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1346, 1346] 00000008:00010000:1.0:1734016144.633157:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016144.633161:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a8673560, flags 0x0 00010000:00010000:1.0:1734016144.633183:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.633191:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [1318912->1323007] (req 1318912->1323007) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.633222:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [1318912->1323007] (req 1318912->1323007) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.633244:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335072128:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016144.633308:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3c00 going to sleep for 11 seconds 00000100:00100000:1.0:1734016144.640566:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335072128:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016144.640594:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [1318912->1323007] (req 1318912->1323007) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.640606:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [1318912->1323007] (req 1318912->1323007) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.640613:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [1318912->1323007] (req 1318912->1323007) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.641099:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.641107:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.641127:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.641138:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 3686916, setting rss=3686916; kms=3686916, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.641148:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.641154:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.641264:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:1.0:1734016144.641273:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be3c00 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.641297:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1318912 1323007) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016144.641320:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59de300 - object 0x240000400:7 offset 1318912<>1323008 00000100:00100000:1.0:1734016144.641335:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335072512:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016144.645643:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be3c00 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.645679:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3c00 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.645717:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335072512:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016144.806913:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335075456:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016144.813285:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016144.813300:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335075584:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016144.815454:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335075456:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016144.817463:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335075584:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016144.822623:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016144.822643:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335075840:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016144.839477:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335075840:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016144.846000:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000008:00010000:1.0:1734016144.846035:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50d83 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4558848->4567039) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994775 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016144.846113:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4558848 4562943) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50d83 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4558848->4567039) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994775 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016144.846248:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59de300 - object 0x280000400:7 offset 2191360<>4562944 00000100:00100000:1.0:1734016144.846278:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335076096:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016144.898160:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50d83 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4558848->4567039) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994775 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.898371:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50d83 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4558848->4567039) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994775 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.898400:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335076096:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016144.978744:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335077120:192.168.201.104@tcp:4:fsx.0 00000008:00010000:1.0:1734016144.992770:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800a86029c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50d83 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4558848->4567039) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994775 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016144.992814:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800a86029c0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3a80/0xd17286ca2ee50d83 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4558848->4567039) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994775 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016144.992837:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335077120:192.168.201.104@tcp:4:fsx.0 00000100:00100000:1.0:1734016145.010596:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335077376:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016145.011903:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335077248:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.019550:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335077376:192.168.201.104@tcp:22:fsx.0 00000080:00200008:1.0:1734016145.020344:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016145.020351:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016145.020375:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016145.020392:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016145.020403:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016145.020416:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016145.020419:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50ccd 00000080:00010000:1.0:1734016145.020424:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50ccd from it ffff8800a872fcd8 00010000:00010000:1.0:1734016145.020426:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016145.020431:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ccd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559946bf expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016145.020454:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016145.020466:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016145.020485:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.020489:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a86731d0, flags 0x400001002 00000008:00010000:1.0:1734016145.020495:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.020496:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672e40, flags 0x400001002 00010000:00010000:1.0:1734016145.020506:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020513:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020522:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020528:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020540:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020546:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020553:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020558:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.020576:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9334215 i_blocks 160 00000020:00010000:1.0:1734016145.020579:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016145.020580:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.020582:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4ba28/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.020585:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.020587:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.020588:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.020591:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016145.020597:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016145.020612:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016145.020615:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016145.020618:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016145.020622:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.020624:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672e40, flags 0x400001002 00000008:00010000:1.0:1734016145.020626:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.020627:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a86731d0, flags 0x400001002 00010000:00010000:1.0:1734016145.020633:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020640:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020647:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020654:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee50da6 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4059136->4071423) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994798 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020663:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020670:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020676:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020682:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097bcc0/0xd17286ca2ee50d1a lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1318912->1323007) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599470c expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.020710:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9334215 i_blocks 160 00000020:00010000:1.0:1734016145.020712:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016145.020714:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.020716:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.020719:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4ba28/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.020720:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.020723:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.020725:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016145.020732:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016145.020737:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 9334215 00000080:00200000:1.0:1734016145.020781:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016145.020785:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016145.020794:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016145.020805:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2173, 2173] 00000008:00010000:1.0:1734016145.020811:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016145.020813:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a86731d0, flags 0x0 00010000:00010000:1.0:1734016145.020832:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020839:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [4706304->4710399] (req 4706304->4710399) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.020875:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [4706304->4710399] (req 4706304->4710399) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.020902:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335077504:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016145.021010:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be33c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016145.036065:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335077504:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016145.036096:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4706304->4710399) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc5599479f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.036121:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4706304->4710399) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc5599479f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016145.036133:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 5139911, setting rss=5139911; kms=5139911, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4706304->4710399) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc5599479f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.036144:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4706304->4710399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599479f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.036150:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ad00/0xd17286ca2ee50dad lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4706304->4710399) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599479f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.036177:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9334215 i_blocks 160 00000100:00100000:1.0:1734016145.036263:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016145.036279:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335077760:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016145.043964:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335077760:192.168.201.104@tcp:22:fsx.0 00010000:00010000:1.0:1734016145.203544:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.288793:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.288808:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.289168:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000008:00010000:1.0:1734016145.289182:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.289227:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (5095424 5099519) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016145.289346:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59df480 - object 0x280000400:7 offset 0<>5099520 00000100:00100000:1.0:1734016145.289368:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335079808:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016145.290919:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000008:00010000:1.0:1734016145.290929:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be36c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.290954:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4190208 4194303) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016145.290975:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59de680 - object 0x280000400:7 offset 4190208<>4194304 00000100:00100000:1.0:1734016145.290989:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335080064:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016145.305526:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be36c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016145.305549:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be36c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.305565:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335080064:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016145.390272:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016145.390596:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be33c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.390619:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335079808:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016145.408590:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335080704:192.168.201.104@tcp:4:fsx.0 00000008:00010000:1.0:1734016145.417671:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800a8602cc0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016145.417751:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800a8602cc0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.417768:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335080704:192.168.201.104@tcp:4:fsx.0 00000100:00100000:1.0:1734016145.687927:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335082880:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016145.699160:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699176:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699188:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699194:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699201:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016145.699211:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5139911, setting rss=5139911; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016145.699221:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699226:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699232:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699238:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1440 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.699242:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699250:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016145.699254:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.699259:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50e39 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.699298:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335082880:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016145.709620:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016145.709629:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.709638:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b5200 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.709644:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.709703:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016145.714631:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016145.714665:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.714671:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016145.725932:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.725949:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50dde lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5095424->5099519) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc559947d0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754759:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1734016145.754766:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50e47 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x979ba2bc55994839 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754773:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b5200 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.754778:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50e47 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x979ba2bc55994839 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.754792:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016145.754802:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016145.754831:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50e47 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994839 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754837:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016145.754854:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50e47 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994839 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754860:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b5200/0xd17286ca2ee50e47 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994839 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754874:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016145.754877:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0000/0xd17286ca2ee50e40 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 147456->151551) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994832 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754882:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a0000 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.754884:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0000/0xd17286ca2ee50e40 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 147456->151551) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994832 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.754893:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016145.754935:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016145.754951:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0000/0xd17286ca2ee50e40 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 147456->151551) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994832 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754956:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016145.754960:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0000/0xd17286ca2ee50e40 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 147456->151551) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994832 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754966:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a0000/0xd17286ca2ee50e40 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 147456->151551) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994832 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.754978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.754979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.754981:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016145.755009:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335084160:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.755051:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016145.755056:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335084032:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.763854:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335084160:192.168.201.104@tcp:103:ldlm_bl.0 00000020:00010000:1.0:1734016145.770038:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a7276218 (ffff88012bf4b1b8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016145.770043:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a7276218 (ffff88012bf4b1b8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.770047:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b3dbba30 (ffff88012bf4ba28/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016145.770055:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.770065:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016145.770076:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a7276218 (ffff88012bf4b1b8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.770078:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b3dbba30 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016145.770092:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016145.770107:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016145.770112:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016145.770127:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016145.770140:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016145.770149:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016145.770160:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016145.770163:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50ca3 00000080:00010000:1.0:1734016145.770167:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50ca3 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016145.770169:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016145.770175:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee50ca3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc5599468e expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016145.770193:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016145.770201:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016145.770214:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.770216:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3030, osc lock ffff8800a8672e40, flags 0x400001002 00000008:00010000:1.0:1734016145.770222:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.770224:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3070, osc lock ffff8800a8673690, flags 0x400001002 00010000:00010000:1.0:1734016145.770242:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50e55 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.770248:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee50e55 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.770289:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.770300:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.770308:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.770313:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.778474:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9334215 i_blocks 176 00000020:00010000:1.0:1734016145.778481:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016145.778484:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.778488:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3030 (ffff88012bf4ba28/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.778492:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3070 (ffff88012bf4b518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.778495:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.778497:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3030 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.778501:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3070 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016145.778511:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016145.778529:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016145.778534:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016145.778540:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016145.778551:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.778553:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3030, osc lock ffff8800a8673690, flags 0x400001002 00000008:00010000:1.0:1734016145.778557:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016145.778558:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3070, osc lock ffff8800a8672e40, flags 0x400001002 00010000:00010000:1.0:1734016145.778577:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee50e5c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.778585:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee50e5c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.778647:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.778655:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.778663:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.778670:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.785447:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9334215 i_blocks 176 00000020:00010000:1.0:1734016145.785453:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_release():233 00000020:00010000:1.0:1734016145.785455:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.785459:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3030 (ffff88012bf4b518/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.785462:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3070 (ffff88012bf4ba28/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016145.785465:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db7a6c8 (ffff88012bf4b1b8/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.785467:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3030 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016145.785470:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3070 (ffff88012bf4b1b8/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016145.785480:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016145.785489:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 9334215 00000080:00200000:1.0:1734016145.785529:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016145.785533:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016145.785544:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016145.785554:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1898, 1898] 00000008:00010000:1.0:1734016145.785565:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016145.785567:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea2b0, osc lock ffff8800a8672e40, flags 0x0 00010000:00010000:1.0:1734016145.785579:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800a9975000 type 11 mode 2 res 9663677440/7 (3579904 3583999) 00010000:00010000:1.0:1734016145.785588:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50e63 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.785594:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50e63 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [3579904->3583999] (req 3579904->3583999) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.785622:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50e63 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [3579904->3583999] (req 3579904->3583999) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.785643:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ded80 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335084672:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016145.785721:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00010000:00010000:1.0:1734016145.790462:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1734016145.790468:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50e32 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x979ba2bc55994824 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.790475:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b73c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.790479:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50e32 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x979ba2bc55994824 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.790494:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016145.790507:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016145.790538:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50e32 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994824 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.790544:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016145.790566:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50e32 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994824 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.790572:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b73c0/0xd17286ca2ee50e32 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994824 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.790599:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016145.790602:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.790608:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b6ac0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.790611:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.790620:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016145.793855:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016145.793904:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 2/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.793914:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016145.793927:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 1/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.793934:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50e2b lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1077248->1081343) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc5599481d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.793948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.793949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.793952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016145.793973:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335084928:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.897904:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335087104:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.904261:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335087104:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016145.912278:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016145.912289:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.912300:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2f40 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.912306:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016145.912327:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016145.912341:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016145.912391:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.912396:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016145.917104:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.917118:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2f40/0xd17286ca2ee50e4e lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4407296->4415487) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994840 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.917175:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.917177:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016145.917180:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016145.917195:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016145.917217:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335087360:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.936469:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335087360:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.940972:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335087744:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016145.941414:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016145.941423:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335087872:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016145.948083:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335087616:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016145.952429:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335087744:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016145.957968:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335087872:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016145.971623:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8749180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335088256:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016145.991249:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991268:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991279:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991286:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991291:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016145.991302:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016145.991324:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991329:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991335:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991342:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a18c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016145.991345:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991352:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016145.991356:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016145.991361:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a18c0/0xd17286ca2ee50eda lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016145.992082:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335088256:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016145.995776:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8749c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335088512:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016146.012252:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012268:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012278:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012283:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012297:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.012308:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016146.012314:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012319:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012325:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012331:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1680 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016146.012334:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012342:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016146.012346:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.012351:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1680/0xd17286ca2ee50ee8 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.012383:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335088512:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.039742:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335088896:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016146.067318:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016146.067324:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067330:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1200 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016146.067334:0:4738:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067340:0:4738:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067346:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067365:0:4738:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067398:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067402:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016146.067434:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067440:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a1200/0xd17286ca2ee50ecc lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc559948cc expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.067464:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.067466:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.067467:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016146.067475:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016146.067491:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335089664:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.072316:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335089664:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.120821:0:4728:0:(service.c:2100:ptlrpc_server_handle_req_in()) unwrap req x1818247338526208 00000100:00100000:1.0:1734016146.120825:0:4728:0:(service.c:2165:ptlrpc_server_handle_req_in()) got req x1818247338526208 00000100:00100000:1.0:1734016146.121180:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 190 00000100:00100000:1.0:1734016146.121184:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff88012e264000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:9511:x1818247338526208:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016146.121191:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ef6 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3526656->3538943) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559948fd expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.121197:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ef6 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3526656->3538943) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559948fd expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.121208:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6cbb8[0x4, 1, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016146.121209:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 4194304 mtime: 1734016146 atime: 1734016145 ctime: 1734016146 blocks: 72 00000020:00010000:1.0:1734016146.121212:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6c3e8[0x4, 2, [0x280000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016146.121213:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 4194304 mtime: 1734016146 atime: 1734016145 ctime: 1734016146 blocks: 72 00000100:00100000:1.0:1734016146.121232:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff88012e264000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:9511:x1818247338526208:12345-192.168.201.104@tcp:106: Request processed in 47us (755us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016146.121237:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 190 00000100:00100000:1.0:1734016146.165897:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335091456:192.168.201.104@tcp:4:fsx.0 00000008:00010000:1.0:1734016146.184444:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800a8602300 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ef6 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3526656->3538943) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc559948fd expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.186612:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800a8602300 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b58c0/0xd17286ca2ee50ef6 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3526656->3538943) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc559948fd expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.186650:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335091456:192.168.201.104@tcp:4:fsx.0 00010000:00010000:1.0:1734016146.206061:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:1.0:1734016146.209209:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50f3c lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1810432->1814527) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994958 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.209307:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1810432 1814527) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50f3c lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1810432->1814527) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994958 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016146.209520:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59df480 - object 0x280000400:7 offset 0<>4194304 00000100:00100000:1.0:1734016146.350827:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874b800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335092224:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016146.350872:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016146.350878:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.350894:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.350938:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016146.350952:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.350956:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3f30, osc lock ffff8800a8673690, flags 0x0 00000008:00010000:1.0:1734016146.350959:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.350960:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3f70, osc lock ffff8800a8672f70, flags 0x0 00010000:00010000:1.0:1734016146.350975:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff880136af2000 type 11 mode 2 res 10737419264/7 (0 18446744073709551615) 00010000:00010000:1.0:1734016146.351001:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50f4a lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.351011:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50f4a lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.351044:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee50f4a lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.351065:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335092480:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.351250:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3180 going to sleep for 11 seconds 00010000:00010000:1.0:1734016146.359080:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1734016146.359089:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50f43 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x979ba2bc5599495f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.359098:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b4fc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016146.359103:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50f43 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x979ba2bc5599495f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.359154:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016146.359166:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016146.359204:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50f43 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc5599495f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.359210:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016146.359231:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50f43 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc5599495f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.359238:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b4fc0/0xd17286ca2ee50f43 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc5599495f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.359274:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016146.359277:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50f3c lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1810432->1814527) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994958 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.359282:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1440 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016146.359286:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1440/0xd17286ca2ee50f3c lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1810432->1814527) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994958 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.359295:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00010000:00010000:1.0:1734016146.405165:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016146.405172:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f0b lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994919 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.405179:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b6880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016146.405184:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f0b lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994919 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.405246:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016146.405260:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016146.405296:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f0b lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994919 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.405301:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016146.405319:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f0b lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994919 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.405325:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f0b lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994919 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.405351:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.405352:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.405353:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016146.405363:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016146.405377:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335092992:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.432261:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335092992:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.437343:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335093504:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016146.448548:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448557:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448565:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448570:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448576:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.448584:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 3843405, setting rss=3843405; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016146.448833:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448841:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448846:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448851:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a0240 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016146.448854:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448859:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016146.448862:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.448866:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee50f5f lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.448899:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335093504:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.545112:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335095168:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.554557:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335095296:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016146.556916:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335095168:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.559336:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335095296:192.168.201.104@tcp:22:fsx.0 00000080:00200000:1.0:1734016146.570453:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 80 00000020:00010000:1.0:1734016146.570460:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.570463:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.570468:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e84b0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.570472:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b60e84f0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.570474:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.570476:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e84b0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.570480:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b60e84f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.570492:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.570507:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016146.570512:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.570518:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.570531:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.570534:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016146.570541:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.570542:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.570562:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50fba lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.570569:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097ba80/0xd17286ca2ee50fba lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.570605:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50fc1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.570611:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309798c0/0xd17286ca2ee50fc1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.582594:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 80 00000020:00010000:1.0:1734016146.582600:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.582603:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.582608:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.582613:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.582615:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.582616:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.582620:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.582630:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.582638:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 3843405 00000080:00200008:1.0:1734016146.582653:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016146.582658:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016146.582675:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016146.582708:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.582717:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016146.582726:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016146.582729:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50f58 00000080:00010000:1.0:1734016146.582733:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50f58 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016146.582736:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.582742:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016146.582759:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.582766:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.582776:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.582779:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.582783:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.582784:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.582799:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee50fc8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.582805:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee50fc8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.582849:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee50fcf lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.582854:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee50fcf lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.603336:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 80 00000020:00010000:1.0:1734016146.603342:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.603344:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.603350:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.603353:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.603356:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.603358:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.603361:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.603373:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.603390:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016146.603395:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.603401:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.603409:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.603412:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016146.603416:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.603417:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.603431:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50fd6 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.603438:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50fd6 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.603482:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50fdd lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.603488:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee50fdd lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.614942:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 80 00000020:00010000:1.0:1734016146.614948:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.614951:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.614954:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.614957:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.614960:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.614961:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.614965:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.614974:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.619110:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 3843405 00000080:00200008:1.0:1734016146.619134:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016146.619140:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016146.619160:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016146.619177:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.619188:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016146.619201:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016146.619203:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee50f82 00000080:00010000:1.0:1734016146.619209:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50f82 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016146.619212:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.619217:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016146.619240:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.619249:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.619265:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.619268:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.619273:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.619274:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.619286:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619294:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619303:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619312:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619325:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619330:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619337:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619342:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.619356:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 80 00000020:00010000:1.0:1734016146.619360:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.619362:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.619364:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.619367:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.619369:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.619371:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.619374:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.619381:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.619395:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016146.619398:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.619405:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.619414:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [112, 112] 00000008:00010000:1.0:1734016146.619420:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.619422:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.619428:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619435:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (458752 462847) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619442:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619447:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.619452:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.619460:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 80 00000020:00010000:1.0:1734016146.623835:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.623839:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.623845:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea230 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.623856:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.623869:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.623886:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.623888:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea230 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.623902:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.623951:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016146.623957:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8),name=f16b.sanityn 00000002:00010000:1.0:1734016146.623974:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016146.623999:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.624010:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016146.624020:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) 00000080:00010000:1.0:1734016146.624023:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afe190c8) for lock 0xd17286ca2ee50f58 00000080:00010000:1.0:1734016146.624027:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee50f58 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016146.624030:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.624036:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016146.624053:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.624061:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.624073:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.624075:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016146.624080:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.624082:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.624100:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee50fe4 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.624106:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee50fe4 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.624152:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50feb lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.624158:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097af40/0xd17286ca2ee50feb lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.636849:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 88 00000020:00010000:1.0:1734016146.636854:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.636856:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.636861:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.636864:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.636865:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.636867:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.636870:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.636881:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.636899:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016146.636904:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.636910:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.636920:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.636921:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.636926:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.636927:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.636943:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50ff2 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.636949:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130978b40/0xd17286ca2ee50ff2 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.636998:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50ff9 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.637004:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff88013097b840/0xd17286ca2ee50ff9 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.663726:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 3843405 i_blocks 88 00000020:00010000:1.0:1734016146.663733:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.663735:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.663740:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.663743:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.663745:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.663747:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.663750:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.663758:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.663767:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 3843405 00000080:00200008:1.0:1734016146.664104:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 3843405, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 5323971, atime 1734016146.000000000, mtime 1734016146.000000000, ctime 1734016146.000000000 00010000:00010000:1.0:1734016146.664133:0:10407:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.664150:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.664191:0:10407:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.664197:0:10407:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016146.664199:0:10407:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a98b4d80/0xd17286ca2ee50f82 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x979ba2bc559949b3 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1734016146.667821:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016146.667838:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59dd880 x1818247335097216/t0(0) o36->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016146.667863:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335097216:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016146.668119:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00010000:00010000:1.0:1734016146.676932:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016146.676938:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.676945:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b6880 already unused, calling callback (ffffffffa107e1e0) 00010000:00010000:1.0:1734016146.676950:0:5236:0:(ldlm_request.c:1579:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.676956:0:5236:0:(ldlm_request.c:1589:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.676962:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.676991:0:5236:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b4480/0xd17286ca2ee50786 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55993fe9 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.677024:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.677029:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016146.677056:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.677062:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8800aa629800 lock: ffff8800a98b6880/0xd17286ca2ee50f58 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x979ba2bc55994982 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.677077:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.677079:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.677080:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016146.687644:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8800b588c800: committing for last_committed 4294968815 gen 1 00000100:00100000:1.0:1734016146.687727:0:10407:0:(client.c:3012:ptlrpc_free_committed()) @@@ stopping search req@ffff8800b59dd880 x1818247335097216/t4294968816(4294968816) o36->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 488/456 e 0 to 0 dl 1734016162 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000100:00100000:1.0:1734016146.687737:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd880 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335097216:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016146.687779:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016146.687785:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.687800:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.687821:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016146.687835:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.687839:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8673430, flags 0x0 00000008:00010000:1.0:1734016146.687841:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.687843:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672be0, flags 0x0 00010000:00010000:1.0:1734016146.687862:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687879:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687891:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687898:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687903:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687913:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687918:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687924:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687929:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.687935:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.700914:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 88 00000020:00010000:1.0:1734016146.700923:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.700927:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.700931:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.700945:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.700958:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.700987:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.700992:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.700999:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.701009:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.701011:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.701017:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.701037:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.701053:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8) current size 5323971, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 5323971, atime 1734016146.000000000, mtime 1734016146.663000000, ctime 1734016146.663000000, rc 0 00000080:00a00000:1.0:1734016146.701092:0:10407:0:(llite_mmap.c:611:ll_file_mmap()) VFS_Op: fid=[0x200000401:0xc:0x0] vma=ffff8800b6f456c0 start=0x7f7d584a8000 end=0x7f7d584a9000 vm_flags=0xfb 00000080:00200000:1.0:1734016146.701105:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.701114:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.701129:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.701131:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.701136:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.701138:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.701156:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701167:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701178:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701184:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701195:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701202:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701208:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.701214:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.701227:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 88 00000020:00010000:1.0:1734016146.701230:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.701232:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.701234:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.701236:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.701239:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.701241:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.701243:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.701249:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.701299:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 1299 00000080:00200000:1.0:1734016146.701306:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.701313:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.701323:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1299, 1299] 00000008:00010000:1.0:1734016146.701331:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.701334:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.701339:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.708844:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1126400 1130495) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.708872:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.708880:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.708890:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000001:00010000:1.0:1734016146.708919:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.708945:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.708948:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.708953:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.708955:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672720, flags 0x400001002 00010000:00010000:1.0:1734016146.708972:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709019:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709025:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709032:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709051:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709056:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709061:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709067:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.709077:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 88 00000020:00010000:1.0:1734016146.709080:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.709082:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.709085:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.709087:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.709090:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.709091:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.709096:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00010000:00010000:1.0:1734016146.709142:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709148:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709155:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 7/1,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709159:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 7/1,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.709163:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.711295:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.711310:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.751275:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.751283:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.751286:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea230 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.751296:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751310:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.751328:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.751331:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea230 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.751355:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.751372:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.751386:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.751400:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [1299, 1299] 00000008:00010000:1.0:1734016146.751414:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.751418:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.751436:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751453:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1126400 1130495) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751462:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751468:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751474:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000001:00010000:1.0:1734016146.751489:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.751496:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.751498:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672720, flags 0x400001002 00000008:00010000:1.0:1734016146.751501:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.751502:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.751512:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751520:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751527:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751538:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751549:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751555:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751562:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751568:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.751578:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 88 00000020:00010000:1.0:1734016146.751581:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.751583:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.751585:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.751588:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.751590:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.751592:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.751594:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000008:00010000:1.0:1734016146.751620:0:10407:0:(osc_cache.c:783:osc_extent_find()) ### extent: ffff880136be3e40 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.751641:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afc89248, count 1, dirtied 1 00000020:00010000:1.0:1734016146.751653:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.751655:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.751657:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea230 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.751660:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.751667:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.751675:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.751676:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea230 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.751683:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.751753:0:10407:0:(file.c:5185:ll_fsync()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8), start 5320704, end 5324799, datasync 1 00000080:00200000:1.0:1734016146.751760:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.751764:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1734016146.751774:0:10407:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be3e40 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.751784:0:10407:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff880136be3e40 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.757218:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (1126400 1130495) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016146.757317:0:10407:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59de680 - object 0x240000400:7 offset 1126400<>1129667 00000080:00200000:1.0:1734016146.781094:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1734016146.781171:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335098112:192.168.201.104@tcp:44:fsx.0 00000100:00100000:1.0:1734016146.781282:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3e40 going to sleep for 11 seconds 00000100:00100000:1.0:1734016146.788248:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335098112:192.168.201.104@tcp:44:fsx.0 00000080:00200000:1.0:1734016146.788293:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.788313:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.816633:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 96 00000080:00200000:1.0:1734016146.816646:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.816723:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016146.816729:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016146.816744:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016146.816755:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016146.816787:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016146.816793:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016146.816801:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016146.816817:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335098368:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.817715:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3e40 going to sleep for 11 seconds 00000100:00100000:1.0:1734016146.824634:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335098368:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016146.824660:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.824674:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.824683:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.826066:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.826091:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.826100:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.826108:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016146.826125:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59de300 x1818247335098368/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016162 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016146.826137:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016146.826152:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.826167:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016146.826239:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016146.826246:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee51000 00000080:00010000:1.0:1734016146.826250:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016146.826253:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee51000 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016146.826256:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.826263:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097a640/0xd17286ca2ee51000 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994a3f expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016146.826300:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.826310:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.826327:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.826330:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016146.826337:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.826338:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.826353:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826364:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826375:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826382:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826399:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826412:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826419:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826425:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.826445:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 96 00000020:00010000:1.0:1734016146.826450:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.826454:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.826457:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.826460:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.826463:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.826465:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.826468:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.826475:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.826492:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016146.826496:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.826499:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.826503:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.826505:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.826507:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.826509:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.826514:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826521:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826529:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826535:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee50fa5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1245184->1253375) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc559949d6 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826545:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826552:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826557:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.826563:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.826571:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 5323971 i_blocks 96 00000020:00010000:1.0:1734016146.826574:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.826576:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.826578:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.826580:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.826582:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.826584:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.826586:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.826590:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.826596:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 5323971 00000080:00200008:1.0:1734016146.828075:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 3843405, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 9082183, atime 1734016146.000000000, mtime 1734016146.000000000, ctime 1734016146.000000000 00000020:00100000:1.0:1734016146.828136:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800aa629800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016146.828147:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59de300 x1818247335098624/t0(0) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016146.828167:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335098624:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016146.828260:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3e40 going to sleep for 11 seconds 00000100:00100000:1.0:1734016146.846930:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8800aa629800: committing for last_committed 4294968812 gen 1 00000100:00100000:1.0:1734016146.846966:0:10407:0:(client.c:3012:ptlrpc_free_committed()) @@@ stopping search req@ffff8800b59de300 x1818247335098624/t4294968818(4294968818) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/456 e 0 to 0 dl 1734016162 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000100:00100000:1.0:1734016146.846987:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59de300 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335098624:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016146.847024:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016146.847029:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.847042:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.847065:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016146.847076:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.847080:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8673430, flags 0x0 00000008:00010000:1.0:1734016146.847083:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.847084:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672be0, flags 0x0 00010000:00010000:1.0:1734016146.847108:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.847120:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.847144:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.848117:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335098880:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.851161:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be39c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016146.864900:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335098880:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016146.864924:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.864933:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.864939:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875139:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875147:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875167:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.875177:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875186:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875192:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875214:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875219:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.875238:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.875254:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335099136:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.875958:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3a80 going to sleep for 11 seconds 00010000:00010000:1.0:1734016146.883457:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016146.883466:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.883474:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a21c0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016146.883479:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.883498:0:5236:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016146.884564:0:5236:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016146.884608:0:5236:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.884614:0:5236:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016146.884641:0:5236:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.884648:0:5236:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a21c0/0xd17286ca2ee50f51 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994974 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.884664:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.884666:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016146.884667:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016146.884677:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016146.884710:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335099264:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016146.892395:0:10407:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8800aa629800: committing for last_committed 4294968061 gen 2 00000100:00100000:1.0:1734016146.892423:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335099136:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016146.892437:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.892451:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.892466:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 1129667, setting rss=1129667; kms=1129667, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.892475:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.892481:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.901447:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 96 00000020:00010000:1.0:1734016146.901453:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.901455:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.901459:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.901467:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901477:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.901488:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.901492:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901498:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.901511:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.901513:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.901516:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.901529:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.901541:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 9082183, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 9082183, atime 1734016146.000000000, mtime 1734016146.827000000, ctime 1734016146.827000000, rc 0 00000080:00a00000:1.0:1734016146.901569:0:10407:0:(llite_mmap.c:611:ll_file_mmap()) VFS_Op: fid=[0x200000401:0xc:0x0] vma=ffff8800b6f456c0 start=0x7f7d584a7000 end=0x7f7d584a9000 vm_flags=0xfb 00000080:00200000:1.0:1734016146.901579:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.901586:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.901594:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.901596:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.901600:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.901601:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.901614:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901624:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901633:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901640:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901648:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901654:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901660:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901665:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.901674:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 96 00000020:00010000:1.0:1734016146.901676:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.901678:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.901681:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.901683:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.901685:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.901687:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.901719:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.901725:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.901760:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 2216 00000080:00200000:1.0:1734016146.901767:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.901774:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.901781:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2216, 2216] 00000008:00010000:1.0:1734016146.901787:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.901789:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.901795:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901806:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4882432 4886527) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901813:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901818:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901824:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901857:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901864:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3690496 3694591) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901870:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901875:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.901880:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.902488:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.902495:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.912443:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.912446:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.912449:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea830 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.912454:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912462:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.912474:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.912476:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea830 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.912490:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.912501:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.912510:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.912519:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2216, 2216] 00000008:00010000:1.0:1734016146.912527:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.912530:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.912542:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912555:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4882432 4886527) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912562:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912568:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912573:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.912597:0:10407:0:(osc_cache.c:783:osc_extent_find()) ### extent: ffff880136be3000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.912608:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 1, dirtied 1 00000020:00010000:1.0:1734016146.912617:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.912618:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.912620:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea830 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.912623:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912628:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.912635:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.912637:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea830 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.912641:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.912668:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 2217 00000080:00200000:1.0:1734016146.912673:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.912677:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.912681:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2217, 2217] 00000008:00010000:1.0:1734016146.912686:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.912688:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.912717:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912723:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4886528 4890623) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912730:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912735:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912740:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000001:00010000:1.0:1734016146.912750:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.912754:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.912755:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.912759:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.912760:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672720, flags 0x400001002 00010000:00010000:1.0:1734016146.912764:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912770:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912775:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 8/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912780:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912787:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912793:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912798:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912803:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.912812:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 96 00000020:00010000:1.0:1734016146.912815:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.912816:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.912818:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.912820:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.912822:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.912824:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.912826:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4b518/1) at cl_lock_fini():85 00010000:00010000:1.0:1734016146.912847:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912853:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4194304 4198399) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912859:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 8/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912864:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 8/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.912868:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.913851:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.913858:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.926735:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.926740:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.926742:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea830 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.926749:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926756:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.926767:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.926769:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea830 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.926783:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.926798:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.926807:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.926817:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2217, 2217] 00000008:00010000:1.0:1734016146.926826:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.926830:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.926847:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926856:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4886528 4890623) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926863:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926868:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926874:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000001:00010000:1.0:1734016146.926885:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.926889:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.926890:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be30f0, osc lock ffff8800a8672720, flags 0x400001002 00000008:00010000:1.0:1734016146.926895:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.926896:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3130, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.926901:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926907:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926912:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 8/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926916:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926924:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926930:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926935:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.926944:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.926955:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 96 00000020:00010000:1.0:1734016146.926957:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.926959:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.926960:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be30f0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.926962:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3130 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.926963:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.926965:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be30f0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.926967:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3130 (ffff88012bf4b518/1) at cl_lock_fini():85 00000008:00010000:1.0:1734016146.926994:0:10407:0:(osc_cache.c:543:osc_extent_merge()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.927000:0:10407:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be30c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.927006:0:10407:0:(osc_cache.c:560:osc_extent_merge()) ### extent: ffff880136be3000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.927013:0:10407:0:(osc_cache.c:771:osc_extent_find()) ### extent: ffff880136be3000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.927026:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 1, dirtied 1 00000020:00010000:1.0:1734016146.927033:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.927035:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.927038:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea830 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.927040:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.927047:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.927053:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.927055:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea830 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.927060:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.927079:0:10407:0:(file.c:5185:ll_fsync()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afe190c8), start 9076736, end 9084927, datasync 1 00000080:00200000:1.0:1734016146.927083:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.927086:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1734016146.927093:0:10407:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be3000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.927102:0:10407:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff880136be3000 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.927132:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4882432 4886527) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016146.927179:0:10407:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dfb80 - object 0x280000400:7 offset 4882432<>4887879 00000080:00200000:1.0:1734016146.936423:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1734016146.936473:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335100544:192.168.201.104@tcp:44:fsx.0 00000100:00100000:1.0:1734016146.937756:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00000100:00100000:1.0:1734016146.941946:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335100544:192.168.201.104@tcp:44:fsx.0 00000080:00200000:1.0:1734016146.941986:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016146.941997:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.948907:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 112 00000080:00200000:1.0:1734016146.948920:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.948969:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016146.948986:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016146.949004:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016146.949017:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 30 res 8589935617/12 (0 0) 00010000:00010000:1.0:1734016146.949056:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016146.949064:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00010000:00010000:1.0:1734016146.949072:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 10407 initiator: MDT0 00000100:00100000:1.0:1734016146.949092:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335100800:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016146.949770:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3000 going to sleep for 11 seconds 00000100:00100000:1.0:1734016146.953661:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job fsx:1c97ffb5-66ce-4359-8d50-0488c17e03ca:10407:1818247335100800:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016146.953686:0:10407:0:(ldlm_request.c:706:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.953771:0:10407:0:(ldlm_request.c:734:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.953780:0:10407:0:(ldlm_lock.c:1060:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.953789:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.953798:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.953807:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1734016146.953813:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1734016146.953822:0:10407:0:(mdc_locks.c:822:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8800b59df800 x1818247335100800/t0(0) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016162 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'fsx.0' uid:0 gid:0 00000002:00010000:1.0:1734016146.953832:0:10407:0:(mdc_locks.c:926:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1734016146.953842:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.953854:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016146.953908:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016146.953915:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee51015 00000080:00010000:1.0:1734016146.953922:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016146.953926:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee51015 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016146.953929:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.953939:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016146.953966:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.954193:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.954211:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.954213:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3030, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016146.954219:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.954220:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3070, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.954237:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee5101c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.954244:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee5101c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.954815:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee51023 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.954821:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee51023 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.969094:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 112 00000020:00010000:1.0:1734016146.969099:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.969102:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.969106:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3030 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.969110:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3070 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.969112:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.969114:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3030 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.969118:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3070 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.969128:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.969146:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016146.969150:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.969159:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.969171:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.969174:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3030, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016146.969179:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.969180:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3070, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016146.969198:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee5102a lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.969207:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee5102a lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.969256:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee51031 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.969260:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee51031 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016146.976305:0:4728:0:(service.c:2100:ptlrpc_server_handle_req_in()) unwrap req x1818247338532480 00000100:00100000:1.0:1734016146.976309:0:4728:0:(service.c:2165:ptlrpc_server_handle_req_in()) got req x1818247338532480 00000100:00100000:1.0:1734016146.976335:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 192 00000100:00100000:1.0:1734016146.976339:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff88012e265880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7685:x1818247338532480:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016146.976345:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.976351:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.976363:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6cbb8[0x4, 294, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016146.976365:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 4887879 mtime: 1734016146 atime: 1734016146 ctime: 1734016146 blocks: 104 00000020:00010000:1.0:1734016146.976367:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b3d6c3e8[0x4, 2, [0x280000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016146.976368:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 4887879 mtime: 1734016146 atime: 1734016146 ctime: 1734016146 blocks: 104 00000100:00100000:1.0:1734016146.976397:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff88012e265880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7685:x1818247338532480:12345-192.168.201.104@tcp:106: Request processed in 56us (516us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016146.976402:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 192 00000080:00200000:1.0:1734016146.979946:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 112 00000020:00010000:1.0:1734016146.979953:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.979955:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.979959:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3030 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.979963:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3070 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.979965:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.979967:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3030 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.979980:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3070 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.979991:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016146.980001:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 9082183 00000080:00200008:1.0:1734016146.980052:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 3307383, type: SEEK_SET 00000080:00200008:1.0:1734016146.980058:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 3307383, type: SEEK_SET, rc = 3307383 00000080:00200008:1.0:1734016146.980066:0:10407:0:(file.c:2466:do_file_write_iter()) START file f16b.sanityn:[0x200000401:0xc:0x0], ppos: 3307383, count: 2559 00000080:00200000:1.0:1734016146.980072:0:10407:0:(file.c:1826:ll_file_io_generic()) f16b.sanityn: write ppos: 3307383, bytes: 2559 00000080:00200000:1.0:1734016146.980077:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000020:00200000:1.0:1734016146.980084:0:10407:0:(cl_io.c:195:cl_io_rw_init()) header@ffff8800b3d6cbb8[0x4, 294, [0x200000401:0xc:0x0] hash] 00000020:00200000:1.0:1734016146.980085:0:10407:0:(cl_io.c:195:cl_io_rw_init()) io range: 2 [3307383, 3309942) 0 0 00000080:00200000:1.0:1734016146.980089:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016146.980099:0:10407:0:(file.c:1893:ll_file_io_generic()) Range lock [807, 808] 00020000:00200000:1.0:1734016146.980105:0:10407:0:(lov_io.c:1021:lov_io_rw_iter_init()) stripe: 0 chunk: [3307383, 3309942) 3309942, 2559 00020000:00200000:1.0:1734016146.980109:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016146.980123:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [807, 808] 00000008:00010000:1.0:1734016146.980136:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016146.980138:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016146.980154:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980163:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3305472 3313663) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980171:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980176:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980181:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.980187:0:10407:0:(vvp_io.c:1326:vvp_io_write_start()) f16b.sanityn: write [3307383, 3309942) 00000080:00200000:1.0:1734016146.980195:0:10407:0:(rw26.c:819:ll_write_begin()) Writing 807 of 1911 to 2185 bytes 00010000:00010000:1.0:1734016146.980240:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980248:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4886528 4890623) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980255:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980262:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980268:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980279:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.980285:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.987204:0:10407:0:(rw26.c:1006:ll_write_end()) pos 3307383, len 2185, copied 2185 00000080:00200000:1.0:1734016146.987211:0:10407:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7731a20[3 ffff8800b3d6cc38 2 1 ffff88012db79058] 00000080:00200000:1.0:1734016146.987214:0:10407:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea00049a0340 2fffff0000080d 3:0 ffff8800a7731a20 807 lru 00000080:00200000:1.0:1734016146.987220:0:10407:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a7731a80 807: 1< 1 - - > 2< 3305472 0 4096 0x0 0x40009 | ffff8800a9985cc8 ffff8800a85d8340 > 3< 0 0 > 4< 0 0 8 489881600 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1734016146.987222:0:10407:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7731a20 00000080:00200000:1.0:1734016146.987223:0:10407:0:(rw26.c:1044:ll_write_end()) queued page: 1. 00000080:00200000:1.0:1734016146.987228:0:10407:0:(rw26.c:819:ll_write_begin()) Writing 808 of 0 to 374 bytes 00000080:00200000:1.0:1734016146.991539:0:10407:0:(rw26.c:1006:ll_write_end()) pos 3309568, len 374, copied 374 00000080:00200000:1.0:1734016146.991548:0:10407:0:(rw26.c:1044:ll_write_end()) page@ffff8800a7731948[3 ffff8800b3d6cc38 2 1 ffff88012db79058] 00000080:00200000:1.0:1734016146.991551:0:10407:0:(rw26.c:1044:ll_write_end()) vmpage @ffffea0004990480 2fffff0000080d 3:0 ffff8800a7731948 808 lru 00000080:00200000:1.0:1734016146.991558:0:10407:0:(rw26.c:1044:ll_write_end()) osc-page@ffff8800a77319a8 808: 1< 1 - - > 2< 3309568 0 4096 0x0 0x40009 | ffff8800a9985cc8 ffff8800a85d8340 > 3< 0 0 > 4< 0 0 8 489881600 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1734016146.991561:0:10407:0:(rw26.c:1044:ll_write_end()) end page@ffff8800a7731948 00000080:00200000:1.0:1734016146.991562:0:10407:0:(rw26.c:1044:ll_write_end()) queued page: 2. 00000080:00200000:1.0:1734016146.991571:0:10407:0:(vvp_io.c:1218:vvp_io_write_commit()) commit async pages: 2, from 1911, to 374 00000008:00010000:1.0:1734016146.991597:0:10407:0:(osc_cache.c:783:osc_extent_find()) ### extent: ffff880136be3e40 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016146.991615:0:10407:0:(osc_cache.c:2323:osc_queue_async_io()) ### extent: ffff880136be3e40 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016146.991626:0:10407:0:(vvp_io.c:1133:vvp_set_batch_dirty()) mapping ffff8800afe19248, count 2, dirtied 2 00000080:00200000:1.0:1734016146.991630:0:10407:0:(vvp_io.c:1241:vvp_io_write_commit()) Committed 2 pages 2559 bytes, tot: 2559 00000080:00200000:1.0:1734016146.991633:0:10407:0:(vvp_io.c:1405:vvp_io_write_start()) f16b.sanityn: write bytes 0, result: 2559 00000020:00010000:1.0:1734016146.991648:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016146.991651:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016146.991655:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800b61ea230 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016146.991663:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.991672:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016146.991688:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016146.991798:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800b61ea230 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016146.991805:0:10407:0:(vvp_io.c:533:vvp_io_advance()) advancing 2559 bytes 00000080:00200000:1.0:1734016146.991810:0:10407:0:(file.c:1930:ll_file_io_generic()) Range unlock [807, 808] 00000080:00200000:1.0:1734016146.991817:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016146.991821:0:10407:0:(file.c:1964:ll_file_io_generic()) f16b.sanityn: 2 io complete with rc: 0, result: 2559, restart: 0 00000080:00200000:1.0:1734016146.991831:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 112 00000080:00200000:1.0:1734016146.991839:0:10407:0:(file.c:2044:ll_file_io_generic()) iot: 2, result: 2559 00000080:00200008:1.0:1734016146.991862:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016146.991870:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016146.991892:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016146.991904:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.991913:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016146.991923:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016146.991925:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee51015 00000080:00010000:1.0:1734016146.991929:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee51015 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016146.991932:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016146.991937:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff88013097ba80/0xd17286ca2ee51015 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994a62 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016146.991955:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016146.991965:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016146.991986:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.991990:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8673430, flags 0x400001002 00000008:00010000:1.0:1734016146.991994:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016146.991996:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8672be0, flags 0x400001002 00010000:00010000:1.0:1734016146.992014:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee51038 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.992020:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff88013097a880/0xd17286ca2ee51038 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.992083:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee5103f lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016146.992089:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff880130979440/0xd17286ca2ee5103f lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016147.002339:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 112 00000020:00010000:1.0:1734016147.002345:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016147.002348:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.002352:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.002356:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.002358:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.002360:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.002362:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016147.002372:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016147.002387:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016147.002392:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016147.002399:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016147.002410:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016147.002412:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016147.002416:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016147.002417:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016147.002432:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee51046 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.002438:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8801309798c0/0xd17286ca2ee51046 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.002480:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5104d lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.002487:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff88013097a1c0/0xd17286ca2ee5104d lrc: 3/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016147.010741:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9082183 i_blocks 112 00000020:00010000:1.0:1734016147.010748:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016147.010751:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.010756:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.010759:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.010762:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.010764:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.010767:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016147.010781:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016147.010790:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 9082183 00000080:00200008:1.0:1734016147.010933:0:10407:0:(llite_lib.c:2284:ll_setattr_raw()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 9082183, valid attrs 2068, mode 8800, uid -1248067584, gid -30720, new size 9450812, atime 1734016147.000000000, mtime 1734016147.000000000, ctime 1734016147.000000000 00000020:00100000:1.0:1734016147.010993:0:10407:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800aa629800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1734016147.011002:0:10407:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800b59df100 x1818247335102592/t0(0) o36->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016147.011025:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335102592:192.168.201.104@tcp:36:fsx.0 00000100:00100000:1.0:1734016147.011662:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3a80 going to sleep for 11 seconds 00000100:00100000:1.0:1734016147.024879:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df100 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335102592:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016147.024918:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016147.024924:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016147.024939:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016147.024957:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016147.024967:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016147.024979:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8673430, flags 0x0 00000008:00010000:1.0:1734016147.024982:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016147.024984:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8672be0, flags 0x0 00010000:00010000:1.0:1734016147.025001:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025012:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025022:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025029:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025035:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025044:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025050:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025056:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025061:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.025071:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.025152:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335102976:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016147.032204:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335102976:192.168.201.104@tcp:10:fsx.0 00000080:00200000:1.0:1734016147.032361:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9450812 i_blocks 112 00000020:00010000:1.0:1734016147.032366:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016147.032368:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.032372:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016147.032379:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032389:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016147.032401:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00010000:00010000:1.0:1734016147.032404:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032411:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016147.032418:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff8800a72766c8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.032419:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.032425:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016147.032439:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016147.032450:0:10407:0:(llite_lib.c:2534:ll_setattr_raw()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afe190c8) current size 9450812, valid attrs 2168, mode 8800, uid -1248067584, gid -30720, new size 9450812, atime 1734016147.000000000, mtime 1734016147.010000000, ctime 1734016147.010000000, rc 0 00000080:00a00000:1.0:1734016147.032477:0:10407:0:(llite_mmap.c:611:ll_file_mmap()) VFS_Op: fid=[0x200000401:0xc:0x0] vma=ffff8800b6f456c0 start=0x7f7d584a6000 end=0x7f7d584a9000 vm_flags=0xfb 00000080:00200000:1.0:1734016147.032487:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016147.032495:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016147.032505:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016147.032507:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8672be0, flags 0x400001002 00000008:00010000:1.0:1734016147.032511:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016147.032512:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8673430, flags 0x400001002 00010000:00010000:1.0:1734016147.032526:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032535:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032544:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032550:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032560:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032567:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032573:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032578:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8801309786c0/0xd17286ca2ee5100e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a54 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016147.032586:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 9450812 i_blocks 112 00000020:00010000:1.0:1734016147.032589:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_release():233 00000020:00010000:1.0:1734016147.032591:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.032593:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3ab0 (ffff8800a87117e8/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.032595:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3af0 (ffff88012bf4bc68/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016147.032596:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78ff8 (ffff88012bf4b518/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.032598:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3ab0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016147.032599:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3af0 (ffff88012bf4b518/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016147.032604:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1734016147.032634:0:10407:0:(rw.c:1991:ll_readpage()) fast read pgno: 2305 00000080:00200000:1.0:1734016147.032641:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1734016147.032647:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016147.032654:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [2305, 2305] 00000008:00010000:1.0:1734016147.032660:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016147.032662:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea230, osc lock ffff8800a8673430, flags 0x0 00010000:00010000:1.0:1734016147.032667:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032673:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (5246976 5251071) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032684:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032767:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032774:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032806:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032812:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (4890624 4894719) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032819:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 8/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032824:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 8/1,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.032830:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.033429:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.033436:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.033548:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:1.0:1734016147.033557:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff880136be3a80 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.033576:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (5246976 5251071) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016147.033610:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dfb80 - object 0x280000400:7 offset 4890624<>5259264 00000100:00100000:1.0:1734016147.033624:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335103232:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016147.047112:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff880136be3a80 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.047186:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff880136be3a80 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee51007 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994a4d expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.047213:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335103232:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016147.080150:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335103872:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016147.095390:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095398:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095405:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095411:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095416:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.095423:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5256508, setting rss=5256508; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016147.095430:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095436:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095441:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095445:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3840 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.095448:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095454:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016147.095457:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.095471:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a3840/0xd17286ca2ee5105b lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.095502:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335103872:192.168.201.104@tcp:101:fsx.0 00000008:00010000:1.0:1734016147.183398:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8180 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.183425:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3829760 3833855) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016147.183458:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dfb80 - object 0x280000400:7 offset 3829760<>3833856 00000100:00100000:1.0:1734016147.194130:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335105792:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016147.205950:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016147.399278:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335107200:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016147.417942:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.417961:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.417981:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.417988:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.417996:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.418009:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 5256508, setting rss=5256508; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016147.418025:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.418035:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.418043:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.418050:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1200 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.418055:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.418066:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016147.418072:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.418077:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a1200/0xd17286ca2ee510bd lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.418122:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335107200:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016147.613138:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749c00 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335107968:192.168.201.104@tcp:36:fsx.0 00000080:00200000:1.0:1734016147.613176:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016147.613181:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016147.613196:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016147.613219:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1734016147.613230:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016147.613233:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3ab0, osc lock ffff8800a8672be0, flags 0x0 00000008:00010000:1.0:1734016147.613236:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016147.613237:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3af0, osc lock ffff8800a8672720, flags 0x0 00010000:00010000:1.0:1734016147.613261:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.613271:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.613296:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.613314:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335108224:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016147.613417:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00010000:00010000:1.0:1734016147.631343:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016147.631352:0:5236:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.631361:0:5236:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a1680 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.631370:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.631428:0:5236:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800a8602e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.631444:0:5236:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff8800a8602e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.631485:0:5236:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3829760 3833855) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016147.631544:0:5236:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59dfb80 - object 0x280000400:7 offset 3829760<>3833856 00000100:00100000:1.0:1734016147.631725:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016147.631741:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335108352:192.168.201.104@tcp:4:fsx.0 00000100:00100000:1.0:1734016147.641277:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335108224:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016147.641307:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.641319:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.641327:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.647744:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800a8602e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.647794:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800a8602e40 ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee5109a lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3829760->3833855) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994afc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.647808:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335108352:192.168.201.104@tcp:4:fsx.0 00010000:00010000:1.0:1734016147.676319:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676327:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676351:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.676360:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 5256508, setting rss=5256508; kms=5256508, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676368:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676374:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff88013097a640/0xd17286ca2ee510d2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b3b expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676393:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800a9975000 type 11 mode 2 res 9663677440/7 (0 18446744073709551615) 00010000:00010000:1.0:1734016147.676404:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676411:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.676435:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.676449:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335108608:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016147.676487:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be30c0 going to sleep for 11 seconds 00000100:00100000:1.0:1734016147.697522:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335108608:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016147.697552:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.697561:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.697568:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.697595:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335108736:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016147.702266:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.702274:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.702296:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.702305:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.702314:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.702319:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.702398:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335109120:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016147.702440:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016147.702445:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335108992:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016147.729394:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335108992:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016147.729408:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335109120:192.168.201.104@tcp:10:fsx.0 00010000:00010000:1.0:1734016147.762219:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016147.762225:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.762231:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff880130978fc0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.762236:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016147.762282:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016147.762293:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016147.762319:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.762325:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016147.762347:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.762352:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff880130978fc0/0xd17286ca2ee510d9 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994b42 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.762376:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016147.762378:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016147.762379:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016147.762384:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016147.762394:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335110144:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016147.782203:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335110272:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016147.880159:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335111040:192.168.201.104@tcp:16:fsx.0 00000100:00100000:1.0:1734016147.888575:0:1970:0:(client.c:2968:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8800b588c800: committing for last_committed 4294968070 gen 2 00000100:00100000:1.0:1734016147.889182:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335111040:192.168.201.104@tcp:16:fsx.0 00000100:00100000:1.0:1734016147.895299:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335111296:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016147.919604:0:1969:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919620:0:1969:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919633:0:1969:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919639:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 6/2,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919644:0:1969:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 5/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.919655:0:1969:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 4194304, setting rss=4194304; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016147.919669:0:1969:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 4/1,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919675:0:1969:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919680:0:1969:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 4/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919686:0:1969:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3840 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.919701:0:1969:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 5/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919715:0:1969:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016147.919719:0:1969:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 3/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.919724:0:1969:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee510fc lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.919758:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335111296:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016147.926065:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335111680:192.168.201.104@tcp:103:ldlm_bl.0 00000008:00010000:1.0:1734016147.934530:0:1970:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8a80 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.934556:0:1970:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3772416 3776511) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016147.934581:0:1970:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59df800 - object 0x280000400:7 offset 3772416<>3776512 00000100:00100000:1.0:1734016147.935269:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016147.935280:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335111936:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016147.940980:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8a80 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016147.941017:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8a80 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016147.941042:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335111936:192.168.201.104@tcp:3:fsx.0 00010000:00010000:1.0:1734016147.963614:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1734016147.963620:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee51111 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x979ba2bc55994b8f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963627:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b7a80 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.963631:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee51111 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x979ba2bc55994b8f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016147.963648:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016147.963658:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016147.963716:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee51111 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994b8f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963721:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016147.963738:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee51111 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994b8f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963744:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b7a80/0xd17286ca2ee51111 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x979ba2bc55994b8f expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963776:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016147.963779:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963788:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a2880 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016147.963791:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016147.963798:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016147.963853:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016147.963870:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 2/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963875:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016147.963881:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 1/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963887:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a87a2880/0xd17286ca2ee5110a lrc: 0/0,0 mode: --/PR res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3772416->3776511) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994b88 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016147.963896:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016147.963897:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016147.963899:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016147.963914:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335112704:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016147.963953:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016147.963969:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335112576:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016147.975437:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335112704:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016147.978857:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335112832:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016147.983459:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335112576:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016147.985118:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335112832:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016148.063638:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335113472:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016148.098297:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016148.098305:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee51142 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5722112->5734399) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994bc7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.098314:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b6ac0 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016148.098320:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee51142 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5722112->5734399) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994bc7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016148.098340:0:4738:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1734016148.098351:0:4738:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1734016148.098395:0:4738:0:(ldlm_request.c:1327:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee51142 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5722112->5734399) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994bc7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.098399:0:4738:0:(ldlm_request.c:1369:_ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1734016148.098968:0:4738:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee51142 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5722112->5734399) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994bc7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.098975:0:4738:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a98b6ac0/0xd17286ca2ee51142 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 5722112->5734399) gid 0 flags: 0x804c69400020020 nid: local remote: 0x979ba2bc55994bc7 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.099007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016148.099008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1734016148.099011:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016148.099020:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016148.099036:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335114112:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016148.114468:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335114112:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016148.239503:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8749180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335115776:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016148.252910:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335115776:192.168.201.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1734016148.262329:0:4738:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1734016148.262335:0:4738:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800420000020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.262343:0:4738:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a98b7840 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016148.262348:0:4738:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800428400020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016148.262365:0:4738:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e89c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016148.262374:0:4738:0:(osc_cache.c:1066:osc_extent_make_ready()) ### extent: ffff8800b60e89c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.262416:0:4738:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (5431296 5435391) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016148.262475:0:4738:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59df800 - object 0x280000400:7 offset 5431296<>5440610 00000100:00100000:1.0:1734016148.263328:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016148.263343:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335116032:192.168.201.104@tcp:4:fsx.0 00000008:00010000:1.0:1734016148.271731:0:1970:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e89c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016148.271776:0:1970:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e89c0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff8800a98b7840/0xd17286ca2ee51157 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 958464->966655) gid 0 flags: 0x800429400020000 nid: local remote: 0x979ba2bc55994bdc expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016148.271792:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335116032:192.168.201.104@tcp:4:fsx.0 00000002:00010000:1.0:1734016148.316514:0:10407:0:(mdc_locks.c:1262:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016148.316532:0:10407:0:(ldlm_lock.c:1516:ldlm_lock_match_with_skip()) ### not matched ns ffff8800b5847c00 type 13 mode 4 res 8589935617/12 (0 0) 00000080:00200000:1.0:1734016148.316614:0:10407:0:(file.c:6447:ll_layout_conf()) [0x200000401:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1734016148.316623:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee511a4 00000080:00010000:1.0:1734016148.316628:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016148.316631:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee511a4 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016148.316635:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016148.316645:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016148.316679:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016148.316685:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016148.316751:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.316755:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672d10, flags 0x400001002 00000008:00010000:1.0:1734016148.316775:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.316777:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a86724c0, flags 0x400001002 00010000:00010000:1.0:1734016148.316789:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316801:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316815:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316823:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316840:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316846:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316855:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316861:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016148.316875:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 1968132 i_blocks 104 00000020:00010000:1.0:1734016148.316880:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_release():233 00000020:00010000:1.0:1734016148.316882:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.316884:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4b758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.316887:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4bb48/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.316890:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.316891:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.316894:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016148.316903:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016148.316924:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016148.316929:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016148.316934:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016148.316940:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.316943:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a86724c0, flags 0x400001002 00000008:00010000:1.0:1734016148.316946:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.316959:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672d10, flags 0x400001002 00010000:00010000:1.0:1734016148.316964:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316971:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316978:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316982:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316989:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.316994:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317000:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317005:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016148.317013:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 1968132 i_blocks 104 00000020:00010000:1.0:1734016148.317016:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_release():233 00000020:00010000:1.0:1734016148.317017:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317019:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4bb48/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317021:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4b758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317023:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.317024:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.317026:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016148.317031:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016148.317036:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 1968132 00000080:00200008:1.0:1734016148.317046:0:10407:0:(file.c:5861:ll_getattr_dentry()) START file f16b.sanityn:[0x200000401:0xc:0x0](ffff8800afc890c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1734016148.317052:0:10407:0:(file.c:5757:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xc:0x0](ffff8800afc890c8),name=f16b.sanityn 00000002:00010000:1.0:1734016148.317067:0:10407:0:(mdc_locks.c:1411:mdc_intent_lock()) (name: ,[0x200000401:0xc:0x0]) in obj [0x200000401:0xc:0x0], intent: getattr flags 0 00010000:00010000:1.0:1734016148.317075:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016148.317080:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00010000:1.0:1734016148.317090:0:10407:0:(dcache.c:297:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) 00000080:00010000:1.0:1734016148.317092:0:10407:0:(llite_internal.h:2012:ll_set_lock_data()) setting l_data to inode [0x200000401:0xc:0x0](ffff8800afc890c8) for lock 0xd17286ca2ee511a4 00000080:00010000:1.0:1734016148.317095:0:10407:0:(dcache.c:194:ll_intent_drop_lock()) releasing lock with cookie 0xd17286ca2ee511a4 from it ffff8800a872fcd8 00010000:00010000:1.0:1734016148.317097:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1734016148.317101:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800b37258c0/0xd17286ca2ee511a4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x979ba2bc55994c37 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1734016148.317110:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016148.317113:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016148.317117:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.317118:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a8672d10, flags 0x400001002 00000008:00010000:1.0:1734016148.317121:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.317122:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a86724c0, flags 0x400001002 00010000:00010000:1.0:1734016148.317126:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317132:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317138:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317143:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317155:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317162:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317168:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317172:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016148.317181:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 1968132 i_blocks 104 00000020:00010000:1.0:1734016148.317184:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_release():233 00000020:00010000:1.0:1734016148.317186:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317191:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4b758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317194:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4bb48/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317195:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.317197:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.317199:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016148.317204:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016148.317209:0:10407:0:(file.c:5051:ll_file_seek()) START file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1734016148.317211:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1734016148.317214:0:10407:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xc:0x0] 00000008:00010000:1.0:1734016148.317218:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.317220:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3e70, osc lock ffff8800a86724c0, flags 0x400001002 00000008:00010000:1.0:1734016148.317222:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1734016148.317223:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff880136be3eb0, osc lock ffff8800a8672d10, flags 0x400001002 00010000:00010000:1.0:1734016148.317227:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317234:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317239:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317244:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8800b588c800 lock: ffff8800a87a2ac0/0xd17286ca2ee5119d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994c29 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317252:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317256:0:10407:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317261:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317267:0:10407:0:(ldlm_lock.c:900:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016148.317272:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 1968132 i_blocks 104 00000020:00010000:1.0:1734016148.317276:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) release lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_release():233 00000020:00010000:1.0:1734016148.317278:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317280:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3e70 (ffff88012bf4bb48/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317284:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: ffff880136be3eb0 (ffff88012bf4b758/1) at cl_lock_cancel():150 00000020:00010000:1.0:1734016148.317286:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff88012db78248 (ffff88012bf4bf38/0) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.317287:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3e70 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000020:00010000:1.0:1734016148.317289:0:10407:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: ffff880136be3eb0 (ffff88012bf4bf38/1) at cl_lock_fini():85 00000080:00200000:1.0:1734016148.317292:0:10407:0:(vvp_io.c:304:vvp_io_fini()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1734016148.317295:0:10407:0:(file.c:5083:ll_file_seek()) COMPLETED file f16b.sanityn:[0x200000401:0xc:0x0], offset: 0, type: (null), rc = 1968132 00000080:00200000:1.0:1734016148.317336:0:10407:0:(file.c:1603:ll_io_set_mirror()) f16b.sanityn: desiginated mirror: 0 00000080:00200000:1.0:1734016148.317340:0:10407:0:(vvp_io.c:1866:vvp_io_init()) [0x200000401:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1734016148.317347:0:10407:0:(lov_io.c:849:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1734016148.317354:0:10407:0:(vvp_io.c:215:vvp_io_one_lock_index()) lock: 1 [338, 340] 00000008:00010000:1.0:1734016148.317360:0:10407:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1734016148.317362:0:10407:0:(osc_lock.c:1238:osc_lock_init()) ### lock ffff8800b61ea830, osc lock ffff8800a8672d10, flags 0x0 00010000:00010000:1.0:1734016148.317376:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317384:0:10407:0:(ldlm_request.c:1018:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [1384448->1396735] (req 1384448->1396735) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.317408:0:10407:0:(ldlm_request.c:1105:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 3/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [1384448->1396735] (req 1384448->1396735) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016148.317426:0:10407:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335116672:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016148.317503:0:10407:0:(client.c:2562:ptlrpc_set_wait()) set ffff880136be3300 going to sleep for 11 seconds 00000100:00100000:1.0:1734016148.333936:0:10407:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job fsx:6ad7014d-1575-48d3-b7ba-eccec1d268ba:10407:1818247335116672:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016148.333974:0:10407:0:(ldlm_resource.c:1684:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [1384448->1396735] (req 1384448->1396735) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.333984:0:10407:0:(ldlm_request.c:273:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [1384448->1396735] (req 1384448->1396735) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.333992:0:10407:0:(ldlm_request.c:302:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,1 mode: --/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [1384448->1396735] (req 1384448->1396735) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.336553:0:10407:0:(ldlm_request.c:185:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1384448->1396735) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.336561:0:10407:0:(ldlm_request.c:788:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1384448->1396735) gid 0 flags: 0x20000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.336580:0:10407:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1384448->1396735) gid 0 flags: 0x10000000020000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016148.336589:0:10407:0:(osc_lock.c:178:osc_lock_lvb_update()) ### acquired size 1968132, setting rss=1968132; kms=1968132, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1384448->1396735) gid 0 flags: 0x10020000020000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.336599:0:10407:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1384448->1396735) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.336605:0:10407:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8800aa629800 lock: ffff880130979200/0xd17286ca2ee511ab lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1384448->1396735) gid 0 flags: 0x810020000020000 nid: local remote: 0x979ba2bc55994c3e expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1734016148.336627:0:10407:0:(file.c:1530:ll_merge_attr_nolock()) [0x200000401:0xc:0x0] updating i_size 6637013 i_blocks 128 00000100:00100000:1.0:1734016148.336681:0:1970:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1734016148.336705:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335116928:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016148.345233:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335116928:192.168.201.104@tcp:22:fsx.0 00000100:00100000:1.0:1734016148.352360:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335117184:192.168.201.104@tcp:101:fsx.0 00010000:00010000:1.0:1734016148.360526:0:1970:0:(ldlm_lock.c:765:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360542:0:1970:0:(ldlm_request.c:635:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360553:0:1970:0:(ldlm_request.c:552:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360560:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360567:0:1970:0:(ldlm_lock.c:907:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016148.360579:0:1970:0:(osc_lock.c:178:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00010000:00010000:1.0:1734016148.360600:0:1970:0:(ldlm_lock.c:819:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 4/1,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360607:0:1970:0:(ldlm_lock.c:877:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360613:0:1970:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 4/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360621:0:1970:0:(ldlm_lockd.c:1942:ldlm_handle_bl_callback()) Lock ffff8800a87a3840 already unused, calling callback (ffffffffa0a66d90) 00010000:00010000:1.0:1734016148.360625:0:1970:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 5/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360635:0:1970:0:(ldlm_request.c:1296:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1734016148.360640:0:1970:0:(ldlm_lockd.c:1952:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 3/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.360648:0:1970:0:(ldlm_lock.c:195:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a3840/0xd17286ca2ee511b9 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016148.360688:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1970:1818247335117184:192.168.201.104@tcp:101:fsx.0 00000100:00100000:1.0:1734016148.460374:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335117824:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016148.517916:0:4728:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.104@tcp, seq: 193 00000100:00100000:1.0:1734016148.517923:0:4728:0:(service.c:2344:ptlrpc_server_handle_request()) Handling RPC req@ffff88012e266680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7684:x1818247338542080:12345-192.168.201.104@tcp:106: 00010000:00010000:1.0:1734016148.517931:0:4728:0:(ldlm_lockd.c:2536:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.517941:0:4728:0:(ldlm_lockd.c:2135:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a1680/0xd17286ca2ee510ee lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994b65 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1734016148.517968:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b369d3e8[0x4, 1, [0x200000401:0xc:0x0] hash] 00000020:00010000:1.0:1734016148.517970:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 0 mtime: 1734016148 atime: 1734016147 ctime: 1734016148 blocks: 0 00000020:00010000:1.0:1734016148.517972:0:4728:0:(cl_object.c:300:cl_object_glimpse()) header@ffff8800b369d578[0x4, 2, [0x240000400:0x7:0x0] hash] 00000020:00010000:1.0:1734016148.517974:0:4728:0:(cl_object.c:300:cl_object_glimpse()) size: 0 mtime: 1734016148 atime: 1734016147 ctime: 1734016148 blocks: 0 00000100:00100000:1.0:1734016148.519501:0:4728:0:(service.c:2396:ptlrpc_server_handle_request()) Handled RPC req@ffff88012e266680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:7684:x1818247338542080:12345-192.168.201.104@tcp:106: Request processed in 1573us (3279us total) trans 0 rc 0/0 00000100:00100000:1.0:1734016148.519509:0:4728:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.104@tcp, seq: 193 00000100:00100000:1.0:1734016148.586178:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335119616:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016148.586229:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016148.586234:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a874ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335119488:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016148.587016:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1970:1818247335119360:192.168.201.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1734016148.591553:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a874ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335119488:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016148.591568:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335119616:192.168.201.104@tcp:10:fsx.0 00000100:00100000:1.0:1734016148.592128:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:1.0:1734016148.592136:0:1969:0:(osc_cache.c:1823:try_to_add_extent_for_io()) ### extent: ffff8800b60e8240 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee511ff lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994ca0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1734016148.592154:0:1969:0:(ldlm_lock.c:1499:ldlm_lock_match_with_skip()) ### matched (3391488 3395583) ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee511ff lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994ca0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1734016148.592178:0:1969:0:(osc_request.c:2005:osc_brw_prep_request()) brw rpc ffff8800b59ded80 - object 0x240000400:7 offset 3391488<>3395584 00000100:00100000:1.0:1734016148.592192:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335119872:192.168.201.104@tcp:3:fsx.0 00000008:00010000:1.0:1734016148.607811:0:1969:0:(osc_cache.c:824:osc_extent_finish()) ### extent: ffff8800b60e8240 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee511ff lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994ca0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1734016148.607835:0:1969:0:(osc_cache.c:484:__osc_extent_remove()) ### extent: ffff8800b60e8240 ns: lustre-OST0000-osc-ffff8800aa629800 lock: ffff8800a87a0240/0xd17286ca2ee511ff lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x979ba2bc55994ca0 expref: -99 pid: 10407 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1734016148.607859:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335119872:192.168.201.104@tcp:3:fsx.0 00000100:00100000:1.0:1734016164.668265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335124224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016168.668132:0:1970:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1970:1818247335124608:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016168.668263:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a8748700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335124864:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016168.668387:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335125248:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016168.668429:0:1969:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1734016168.668436:0:1969:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335125120:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016168.699204:0:1970:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8749500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1970:1818247335124608:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016168.699309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335125376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016168.699371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335125504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016168.699394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335125632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016168.699433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335125760:192.168.201.104@tcp:8:kworker.0 00000100:00080000:1.0:1734016174.699015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699023:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016174.699026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699030:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016174.699032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699037:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016174.699040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699043:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016174.699046:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699049:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016174.699051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1734016174.699058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016174.699065:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 434 00000100:00080000:1.0:1734016174.699075:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016174.699083:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016174.699094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016174.699125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ded80 x1818247335125888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016174.699161:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016174.699165:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016178.699121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1734016178.699132:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016178.699140:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 438 00000100:00080000:1.0:1734016178.699149:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016178.699156:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1734016178.699191:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ddf80 x1818247335126016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016178.699226:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699230:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1734016178.699231:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016178.699235:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 438 00000100:00080000:1.0:1734016178.699239:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016178.699242:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016178.699251:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016178.699261:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59dea00 x1818247335126144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016178.699272:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699274:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1734016178.699276:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016178.699280:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 438 00000100:00080000:1.0:1734016178.699282:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016178.699286:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016178.699290:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:1.0:1734016178.699298:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ddc00 x1818247335126272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016178.699306:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699309:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1734016178.699310:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016178.699314:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 438 00000100:00080000:1.0:1734016178.699325:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016178.699329:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016178.699334:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016178.699341:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59de680 x1818247335126400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016178.699350:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699352:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1734016178.699354:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016178.699357:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 438 00000100:00080000:1.0:1734016178.699360:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016178.699364:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016178.699368:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016178.699375:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a8749180 x1818247335126528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016178.699384:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699388:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016178.699391:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016178.699393:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1734016178.699395:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016178.699398:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 438 00000100:00080000:1.0:1734016178.699401:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1734016178.699405:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016178.699410:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016178.699417:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a8749500 x1818247335126656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016179.701822:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701831:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016179.701835:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701838:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016179.701841:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701844:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016179.701847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701850:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016179.701852:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701855:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016179.701857:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701860:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016179.701863:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016179.701866:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709809:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709818:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709821:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709826:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709829:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709832:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709834:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709837:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709840:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709843:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709845:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709848:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016183.709851:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016183.709853:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:1.0:1734016184.667841:0:1969:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1734016168/real 1734016168] req@ffff8800b59cad80 x1818247335125120/t0(0) o400->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1734016184 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016184.686132:0:1969:0:(client.c:2407:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8800b59cad80 x1818247335125120/t0(0) o400->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1734016184 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016184.686145:0:1969:0:(import.c:232:ptlrpc_set_import_discon()) mdc: import ffff8800a98c0800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1734016184.686152:0:1969:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1734016168/real 1734016168] req@ffff8800b59c9180 x1818247335125248/t0(0) o400->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1734016184 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016184.686165:0:1969:0:(client.c:2407:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8800b59c9180 x1818247335125248/t0(0) o400->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1734016184 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016184.686178:0:1969:0:(import.c:232:ptlrpc_set_import_discon()) osc: import ffff8800a98c7000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1734016184.686183:0:1969:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1734016168/real 1734016168] req@ffff8800a8748700 x1818247335124864/t0(0) o400->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1734016184 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016184.686194:0:1969:0:(client.c:2407:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8800a8748700 x1818247335124864/t0(0) o400->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1734016184 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016184.686201:0:1969:0:(import.c:232:ptlrpc_set_import_discon()) osc: import ffff8800aa64a800 already not connected (conn 4, was 2): CONNECTING 00000100:00100000:1.0:1734016184.686213:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335125120:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016184.686220:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1969:1818247335125248:192.168.201.104@tcp:400:kworker.0 00000100:00100000:1.0:1734016193.699228:0:1969:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a8748700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1969:1818247335124864:192.168.201.104@tcp:400:kworker.0 00000800:00000100:2.1:1734016208.264851:0:1960:0:(socklnd_cb.c:2434:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.201.104@tcp (192.168.201.104:988) the network or that node may be down. 00000100:00080000:1.0:1734016213.699130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1734016213.699142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016213.699149:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 463 00000100:00080000:1.0:1734016213.699158:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1734016213.699166:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1734016213.699197:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a8748700 x1818247335127808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016213.699232:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699236:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1734016213.699238:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016213.699242:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 463 00000100:00080000:1.0:1734016213.699246:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1734016213.699250:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016213.699260:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016213.699272:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a8749500 x1818247335127936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016213.699283:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699286:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1734016213.699287:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016213.699291:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 463 00000100:00080000:1.0:1734016213.699294:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1734016213.699297:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016213.699302:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:1.0:1734016213.699310:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a8749180 x1818247335128064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016213.699318:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699321:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1734016213.699322:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016213.699326:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 463 00000100:00080000:1.0:1734016213.699329:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1734016213.699332:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016213.699336:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016213.699345:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7442680 x1818247335128192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016213.699354:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699358:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1734016213.699359:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016213.699363:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 463 00000100:00080000:1.0:1734016213.699367:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1734016213.699370:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016213.699374:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016213.699381:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7441180 x1818247335128320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016213.699393:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699397:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016213.699399:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016213.699402:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1734016213.699403:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016213.699407:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 463 00000100:00080000:1.0:1734016213.699410:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1734016213.699413:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016213.699417:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016213.699424:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7442a00 x1818247335128448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016214.701820:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701828:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016214.701832:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701864:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016214.701866:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701870:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016214.701872:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701875:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016214.701877:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701880:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016214.701883:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701885:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016214.701888:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016214.701891:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709871:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709879:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709883:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709887:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709889:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709893:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709895:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709898:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709902:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709905:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709908:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709911:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016218.709913:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016218.709916:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000400:00000100:1.0:1734016219.715936:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.104:988 00000400:00000100:1.0:1734016219.715987:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016219.716006:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016219.716037:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016219.716045:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016219.716052:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016219.716059:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016219.716064:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016219.716070:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00080000:1.0:1734016219.716250:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716256:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016219.716259:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716263:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016219.716265:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716268:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016219.716270:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716274:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016219.716277:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716281:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1734016219.716284:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716290:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 483 00000100:00080000:1.0:1734016219.716295:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716302:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016219.716313:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016219.716342:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7441f80 x1818247335128576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016219.716358:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716362:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016219.716365:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716367:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1734016219.716369:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716373:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 483 00000100:00080000:1.0:1734016219.716376:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716379:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016219.716384:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016219.716392:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7442300 x1818247335128704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016219.716405:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716409:0:9726:0:(pinger.c:272:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1734016219.716412:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716414:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1734016219.716416:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716419:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 483 00000100:00080000:1.0:1734016219.716422:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716425:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016219.716429:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016219.716437:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7440a80 x1818247335128832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016219.716445:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716447:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1734016219.716449:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716453:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 483 00000100:00080000:1.0:1734016219.716455:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716459:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016219.716463:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:1.0:1734016219.716470:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7443480 x1818247335128960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016219.716478:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716481:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1734016219.716482:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716485:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 483 00000100:00080000:1.0:1734016219.716489:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716492:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016219.716496:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016219.716503:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7440380 x1818247335129088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016219.716512:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716515:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1734016219.716516:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716519:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 479 00000100:00080000:1.0:1734016219.716522:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716525:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1734016219.716529:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016219.716538:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7443100 x1818247335129216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016219.716548:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1734016219.716551:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1734016219.716552:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734016219.716555:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 483 00000100:00080000:1.0:1734016219.716558:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734016219.716562:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1734016219.716570:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a7441500 x1818247335129344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000400:00000100:1.0:1734016227.731964:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.104:988 00000400:00000100:1.0:1734016227.732012:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016227.732031:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016227.732067:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016227.732075:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016227.732082:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016227.732089:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016227.732095:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016227.732101:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000400:00000100:1.0:1734016230.737971:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.104:988 00000400:00000100:1.0:1734016230.738015:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016230.738033:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016230.738066:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016230.738074:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016230.738080:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016230.738087:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016230.738093:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016230.738099:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000400:00000100:1.0:1734016247.028064:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.104:988 00000400:00000100:1.0:1734016247.028133:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016247.028158:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016247.028201:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016247.028210:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016247.028219:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016247.028227:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016247.028234:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016247.028241:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000400:00000100:1.0:1734016267.796103:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734016267.796168:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016267.796194:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016267.796248:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016267.796263:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016267.796275:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016267.796287:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016267.796299:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016267.796309:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:2.0:1734016299.845905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335142912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016299.845941:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016299.845945:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016299.845947:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016299.845951:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 564 00000100:00080000:2.0:1734016299.845954:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016299.845958:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016299.845968:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016299.845978:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335143040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016299.845987:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016299.845991:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016299.845992:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016299.845996:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 564 00000100:00080000:2.0:1734016299.845999:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016299.846002:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016299.846006:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016299.846014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335143168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016299.846023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016299.846026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016299.846028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016299.846031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 564 00000100:00080000:2.0:1734016299.846034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016299.846038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016299.846042:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016299.846050:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335143296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016299.846060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016299.846063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016299.846064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016299.846068:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 564 00000100:00080000:2.0:1734016299.846071:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016299.846074:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016299.846078:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016299.846086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335143424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016299.846094:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016299.846098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016299.846099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016299.846103:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 564 00000100:00080000:2.0:1734016299.846105:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016299.846108:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016299.846112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016299.846120:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335143552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016299.846127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016299.846131:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016299.846132:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016299.846136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 564 00000100:00080000:2.0:1734016299.846139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016299.846142:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016299.846146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016299.846153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335143680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016301.523958:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016302.525952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1734016302.852040:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734016302.852097:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734016302.852115:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016302.852152:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016302.852160:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016302.852166:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016302.852172:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016302.852180:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016302.852190:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00010000:00010000:2.0:1734016304.529890:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016304.853850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.853864:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016304.853871:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.853883:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.853894:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.853905:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016304.853948:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335143808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016304.853986:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.853990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016304.853991:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.853996:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.853999:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.854003:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016304.854023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016304.854033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335143936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016304.854042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.854045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016304.854047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.854050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.854052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.854055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016304.854059:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016304.854067:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335144064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016304.854075:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.854078:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016304.854079:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.854083:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.854085:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.854088:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016304.854092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016304.854101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335144192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016304.854110:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.854112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016304.854114:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.854117:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.854120:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.854123:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016304.854128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016304.854140:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335144320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016304.854153:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.854157:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016304.854159:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.854164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.854169:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.854175:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016304.854182:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016304.854191:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335144448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016304.854200:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016304.854203:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016304.854204:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016304.854207:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 569 00000100:00080000:2.0:1734016304.854210:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016304.854214:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016304.854217:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016304.854224:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335144576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016305.531942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016307.535966:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016308.537998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016309.861835:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.861845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016309.861849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.861860:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.861875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.861883:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016309.861919:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335144704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016309.861952:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.861956:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016309.861957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.861962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.861965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.861969:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016309.861979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016309.861990:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335144832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016309.862000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.862003:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016309.862005:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.862009:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.862011:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.862015:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016309.862019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016309.862027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335144960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016309.862036:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.862038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016309.862040:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.862043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.862046:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.862049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016309.862053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016309.862063:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335145088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016309.862078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.862081:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016309.862083:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.862086:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.862089:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.862092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016309.862096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016309.862103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335145216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016309.862111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.862114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016309.862115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.862119:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.862121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.862125:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016309.862129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016309.862136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335145344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016309.862144:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016309.862147:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016309.862148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016309.862151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 574 00000100:00080000:2.0:1734016309.862155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016309.862158:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016309.862162:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016309.862169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335145472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016310.541935:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016313.547955:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016314.869802:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.869812:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016314.869816:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.869826:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.869835:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.869843:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016314.869878:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335145600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016314.869910:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.869914:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016314.869916:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.869920:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.869923:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.869926:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016314.869935:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016314.869945:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335145728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016314.869962:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.869965:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016314.869966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.869970:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.869972:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.869976:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016314.869980:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016314.869987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335145856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016314.869995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.869998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016314.870000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.870003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.870006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.870009:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016314.870013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016314.870021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335145984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016314.870030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.870033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016314.870035:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.870038:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.870041:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.870044:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016314.870048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016314.870056:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335146112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016314.870064:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.870066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016314.870068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.870071:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.870074:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.870078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016314.870082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016314.870089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335146240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016314.870097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016314.870100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016314.870101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016314.870105:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 579 00000100:00080000:2.0:1734016314.870107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016314.870110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016314.870114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016314.870122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335146368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016316.553952:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016319.559984:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016319.877808:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.877817:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016319.877822:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.877832:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.877842:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.877850:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016319.877885:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335146496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016319.877918:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.877922:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016319.877924:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.877928:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.877931:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.877935:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016319.877944:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016319.877954:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335146624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016319.877964:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.877967:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016319.877969:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.877972:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.877975:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.877979:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016319.877984:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016319.877992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335146752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016319.878000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.878003:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016319.878005:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.878008:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.878010:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.878018:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016319.878022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016319.878031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335146880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016319.878040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.878043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016319.878045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.878049:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.878051:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.878054:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016319.878058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016319.878095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335147008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016319.878104:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.878108:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016319.878110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.878113:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.878124:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.878128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016319.878133:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016319.878140:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335147136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016319.878149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016319.878152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016319.878153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016319.878157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 584 00000100:00080000:2.0:1734016319.878160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016319.878163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016319.878167:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016319.878174:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335147264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016322.565972:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016324.885828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.885839:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016324.885843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.885853:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.885862:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.885870:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016324.885905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335147392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016324.885937:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.885941:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016324.885943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.885948:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.885951:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.885954:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016324.885964:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016324.885974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335147520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016324.886022:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.886026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016324.886028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.886031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.886034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.886038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016324.886043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016324.886052:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335147648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016324.886062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.886065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016324.886066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.886071:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.886073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.886076:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016324.886080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016324.886090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335147776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016324.886098:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.886101:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016324.886102:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.886106:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.886109:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.886113:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016324.886117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016324.886125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335147904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016324.886132:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.886135:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016324.886137:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.886140:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.886152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.886156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016324.886160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016324.886167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335148032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016324.886176:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016324.886179:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016324.886180:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016324.886184:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 589 00000100:00080000:2.0:1734016324.886187:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016324.886190:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016324.886193:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016324.886201:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335148160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016325.572021:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016328.577991:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016329.893849:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.893861:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016329.893866:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.893877:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.893887:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.893937:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016329.893974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335148288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016329.894007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.894011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016329.894013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.894017:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.894021:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.894026:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016329.894036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016329.894046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335148416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016329.894056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.894059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016329.894061:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.894064:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.894067:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.894070:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016329.894074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016329.894082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335148544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016329.894091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.894093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016329.894095:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.894099:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.894101:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.894104:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016329.894108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016329.894117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335148672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016329.894125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.894129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016329.894131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.894135:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.894138:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.894141:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016329.894145:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016329.894152:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335148800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016329.894160:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.894163:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016329.894164:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.894167:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.894170:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.894173:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016329.894177:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016329.894184:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335148928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016329.894192:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016329.894195:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016329.894196:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016329.894200:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 594 00000100:00080000:2.0:1734016329.894202:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016329.894205:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016329.894210:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016329.894217:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335149056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016331.641895:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016334.648002:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016334.901854:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.901863:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016334.901867:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.901877:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.901886:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.901894:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016334.901928:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335149184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016334.901960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.901964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016334.901966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.901971:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.901974:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.901977:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016334.901987:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016334.901997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335149312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016334.902007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.902010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016334.902013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.902016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.902019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.902022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016334.902026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016334.902034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335149440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016334.902042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.902045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016334.902047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.902050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.902053:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.902056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016334.902060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016334.902069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335149568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016334.902078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.902081:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016334.902083:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.902087:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.902089:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.902092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016334.902096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016334.902104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335149696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016334.902113:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.902116:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016334.902117:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.902120:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.902124:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.902127:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016334.902131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016334.902138:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335149824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016334.902146:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016334.902149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016334.902151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016334.902154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 599 00000100:00080000:2.0:1734016334.902157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016334.902160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016334.902163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016334.902170:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335149952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016337.653945:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016339.909828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.909839:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016339.909844:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.909854:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.909863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.909871:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016339.909904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335150080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016339.909936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.909941:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016339.909943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.909947:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.909950:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.909954:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016339.909963:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016339.909973:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335150208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016339.909983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.909986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016339.909988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.909992:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.909994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.909998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016339.910002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016339.910010:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335150336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016339.910018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.910023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016339.910025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.910028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.910031:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.910034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016339.910038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016339.910047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335150464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016339.910056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.910059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016339.910060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.910064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.910066:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.910069:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016339.910073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016339.910080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335150592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016339.910089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.910092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016339.910093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.910096:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.910099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.910102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016339.910107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016339.910114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335150720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016339.910122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016339.910125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016339.910126:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016339.910130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 604 00000100:00080000:2.0:1734016339.910132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016339.910135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016339.910139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016339.910146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335150848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016340.659978:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016343.665954:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016344.917827:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.917837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016344.917841:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.917852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.917861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.917869:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016344.917904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335150976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016344.917936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.917940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016344.917943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.917947:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.917951:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.917954:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016344.917964:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016344.917974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335151104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016344.917983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.917986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016344.917988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.917991:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.917994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.917998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016344.918002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016344.918010:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335151232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016344.918018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.918021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016344.918022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.918026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.918028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.918031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016344.918035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016344.918044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335151360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016344.918053:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.918056:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016344.918058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.918061:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.918064:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.918067:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016344.918071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016344.918078:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335151488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016344.918086:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.918089:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016344.918091:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.918094:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.918098:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.918101:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016344.918105:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016344.918112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335151616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016344.918120:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016344.918123:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016344.918125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016344.918129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 609 00000100:00080000:2.0:1734016344.918132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016344.918135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016344.918139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016344.918146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335151744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016346.671981:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016349.677945:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016349.925824:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.925833:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016349.925837:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.925847:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.925857:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.925864:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016349.925899:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335151872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016349.925932:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.925936:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016349.925938:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.925942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.925945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.925949:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016349.925959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016349.925969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335152000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016349.925986:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.925989:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016349.925991:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.925994:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.925997:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.926000:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016349.926004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016349.926012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335152128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016349.926020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.926023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016349.926025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.926028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.926030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.926034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016349.926038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016349.926046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335152256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016349.926055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.926058:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016349.926060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.926063:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.926066:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.926069:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016349.926074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016349.926081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335152384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016349.926089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.926092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016349.926093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.926096:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.926099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.926102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016349.926106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016349.926113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335152512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016349.926121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016349.926123:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016349.926125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016349.926128:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 614 00000100:00080000:2.0:1734016349.926131:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016349.926134:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016349.926138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016349.926144:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335152640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016352.683954:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016354.933846:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.933855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016354.933859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.933868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.933877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.933885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016354.933920:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335152768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016354.933954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.933958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016354.933960:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.933964:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.933967:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.933971:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016354.933981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016354.933991:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335152896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016354.934001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.934004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016354.934006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.934010:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.934012:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.934017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016354.934021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016354.934029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335153024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016354.934038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.934040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016354.934042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.934045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.934048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.934051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016354.934055:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016354.934063:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335153152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016354.934072:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.934075:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016354.934077:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.934080:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.934083:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.934086:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016354.934090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016354.934097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335153280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016354.934106:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.934110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016354.934111:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.934114:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.934117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.934121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016354.934125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016354.934132:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335153408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016354.934140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016354.934143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016354.934144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016354.934148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 619 00000100:00080000:2.0:1734016354.934150:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016354.934153:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016354.934157:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016354.934164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335153536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016355.689961:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016358.695980:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016359.941836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.941845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016359.941849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.941860:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.941869:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.941878:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016359.941914:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335153664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016359.941947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.941951:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016359.941954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.941958:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.941961:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.941964:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016359.941974:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016359.941985:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335153792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016359.941994:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.941998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016359.941999:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.942003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.942006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.942009:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016359.942014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016359.942021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335153920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016359.942030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.942033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016359.942035:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.942039:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.942041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.942045:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016359.942049:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016359.942058:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335154048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016359.942067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.942070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016359.942071:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.942075:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.942077:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.942081:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016359.942085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016359.942092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335154176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016359.942099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.942103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016359.942104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.942107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.942110:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.942113:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016359.942117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016359.942124:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335154304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016359.942133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016359.942136:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016359.942137:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016359.942141:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 624 00000100:00080000:2.0:1734016359.942152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016359.942156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016359.942160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016359.942167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335154432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016361.701968:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016364.707925:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016364.949842:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.949851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016364.949856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.949866:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.949874:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.949882:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016364.949918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335154560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016364.949950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.949955:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016364.949956:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.949961:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.949964:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.949968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016364.949978:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016364.949988:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335154688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016364.949999:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.950002:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016364.950004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.950007:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.950010:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.950013:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016364.950018:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016364.950026:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335154816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016364.950034:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.950037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016364.950038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.950042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.950044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.950047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016364.950051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016364.950060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335154944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016364.950069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.950073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016364.950075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.950078:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.950081:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.950084:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016364.950088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016364.950096:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335155072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016364.950104:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.950106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016364.950108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.950111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.950114:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.950117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016364.950121:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016364.950128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335155200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016364.950135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016364.950138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016364.950140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016364.950143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 629 00000100:00080000:2.0:1734016364.950145:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016364.950149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016364.950153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016364.950160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335155328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016367.713976:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016367.956496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86dce00 x1818247335154944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335154944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335154944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016367.956573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86dea00 x1818247335155072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956593:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335155072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956605:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956609:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956615:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335155072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016367.956625:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86dd180 x1818247335155200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956645:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335155200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956657:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956661:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956667:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335155200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016367.956677:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86de300 x1818247335155328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956815:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335155328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956824:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956827:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956831:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335155328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016367.956838:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86dc000 x1818247335154560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956849:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335154560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956857:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956860:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956863:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335154560:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016367.956869:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86ddc00 x1818247335154688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956887:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335154688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956895:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956898:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956901:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335154688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016367.956907:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016364/real 1734016367] req@ffff8800a86df800 x1818247335154816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016367.956917:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335154816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016367.956924:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016367.956927:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016367.956930:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335154816:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734016369.957804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016369.957812:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016369.957817:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.957827:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.957836:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.957843:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016369.957878:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335155456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016369.957910:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016369.957915:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016369.957917:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.957921:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.957924:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.957928:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016369.957937:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016369.957947:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335155584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016369.957957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016369.957960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016369.957961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.957966:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.957968:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.957972:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016369.957976:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016369.957984:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335155712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016369.957992:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016369.957995:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016369.957997:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.958000:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.958002:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.958006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016369.958009:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016369.958018:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335155456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016369.958018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335155840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016369.958026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016369.958029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016369.958031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.958034:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.958037:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.958040:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016369.958044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016369.958051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335155968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016369.958060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016369.958063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016369.958065:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.958068:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.958079:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.958083:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734016369.958084:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734016369.958087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016369.958094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335156096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016369.958102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016369.958105:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86dfb80 x1818247335155456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016369.958105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016369.958107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016369.958111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 634 00000100:00080000:2.0:1734016369.958113:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016369.958116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016369.958120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016369.958125:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335155456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016369.958127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335156224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016369.958136:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958139:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958144:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335155456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016369.958152:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335155584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016369.958168:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016369.958175:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86ddf80 x1818247335155584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016369.958186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335155584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016369.958193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958196:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335155584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016369.958205:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335155712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016369.958265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016369.958273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86dc380 x1818247335155712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016369.958284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335155712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016369.958291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958297:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335155712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016369.958319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335155840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016369.958332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016369.958339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86ded80 x1818247335155840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016369.958350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335155840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016369.958357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335155840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016369.958368:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335155968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016369.958380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016369.958385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86de680 x1818247335155968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016369.958395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335155968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016369.958402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335155968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016369.958413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016369.958424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016369.958430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86dd500 x1818247335156096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016369.958440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335156096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016369.958448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016369.958458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016369.958470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016369.958476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016369/real 1734016369] req@ffff8800a86df480 x1818247335156224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016369.958486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335156224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016369.958492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016369.958494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016369.958497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156224:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016370.720002:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016373.725950:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016374.965826:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016374.965835:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016374.965839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.965851:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.965860:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.965867:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016374.965903:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335156352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016374.965937:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016374.965941:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016374.965943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.965948:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.965951:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.965954:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016374.965963:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016374.965974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335156480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016374.965983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016374.965986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016374.965988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.965992:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.965994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.965998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016374.966002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016374.966009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335156608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016374.966018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016374.966021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016374.966023:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.966026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.966029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.966032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016374.966036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016374.966045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335156736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016374.966054:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335156352:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016374.966054:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016374.966057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016374.966058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.966062:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.966064:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.966068:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016374.966072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016374.966079:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335156864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016374.966087:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016374.966090:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016374.966091:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.966094:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.966097:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.966100:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016374.966105:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016374.966112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335156992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016374.966120:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734016374.966155:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016374.966162:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016374.966164:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016374.966168:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 639 00000100:00080000:2.0:1734016374.966172:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016374.966176:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86dca80 x1818247335156352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734016374.966180:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016374.966190:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335157120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016374.966199:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335156352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966210:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966213:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966217:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335156352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016374.966225:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335156480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016374.966248:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966255:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86dc700 x1818247335156480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016374.966267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335156480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335156480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016374.966286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335156608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016374.966299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86df100 x1818247335156608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016374.966315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335156608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335156608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016374.966352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335156736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016374.966365:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86dd880 x1818247335156736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016374.966388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335156736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335156736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016374.966406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016374.966417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86df480 x1818247335156864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016374.966432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335156864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016374.966450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016374.966461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86dd500 x1818247335156992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016374.966476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335156992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335156992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016374.966499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335157120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016374.966510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016374.966516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016374/real 1734016374] req@ffff8800a86de680 x1818247335157120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016374.966525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335157120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016374.966532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016374.966534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016374.966537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335157120:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016376.731940:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016379.737962:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016379.973798:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.973807:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016379.973811:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.973822:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.973830:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.973837:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016379.973872:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335157248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016379.973905:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.973909:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016379.973912:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.973917:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.973920:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.973924:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016379.973933:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016379.973943:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335157376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016379.973953:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.973957:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016379.973958:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.973962:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.973964:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.973967:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016379.973972:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016379.973979:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335157504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016379.973988:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.973991:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016379.973992:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.973995:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.973998:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.974001:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016379.974005:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016379.974014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335157632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016379.974023:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335157248:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016379.974023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.974026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016379.974028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.974032:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.974034:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.974064:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016379.974068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016379.974077:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335157760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016379.974086:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.974089:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016379.974090:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.974094:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.974096:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.974100:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016379.974103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016379.974111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335157888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016379.974118:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016379.974118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016379.974121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016379.974123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016379.974127:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 644 00000100:00080000:2.0:1734016379.974130:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016379.974133:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016379.974137:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016379.974141:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86ded80 x1818247335157248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016379.974144:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335158016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016379.974161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335157248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974172:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974175:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974180:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335157248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016379.974188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335157376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016379.974207:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016379.974214:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86dc380 x1818247335157376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016379.974226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335157376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335157376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016379.974245:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335157504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016379.974258:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016379.974264:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86ddf80 x1818247335157504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016379.974274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335157504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335157504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016379.974312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335157632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016379.974325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016379.974331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86dfb80 x1818247335157632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016379.974348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335157632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335157632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016379.974366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335157760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016379.974378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016379.974384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86df800 x1818247335157760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016379.974394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335157760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335157760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016379.974411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335157888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016379.974423:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016379.974429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86ddc00 x1818247335157888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016379.974439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335157888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335157888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016379.974458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016379.974470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016379.974475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016379/real 1734016379] req@ffff8800a86dc000 x1818247335158016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016379.974485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335158016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016379.974492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016379.974494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016379.974497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158016:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016382.743964:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016384.981828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.981837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016384.981842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.981852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 649 00000100:00080000:2.0:1734016384.981861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.981869:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016384.981904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335158144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016384.981936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.981940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016384.981942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.981946:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 649 00000100:00080000:2.0:1734016384.981949:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.981978:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016384.981989:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016384.982000:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335158272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016384.982010:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.982014:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016384.982016:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.982019:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 649 00000100:00080000:2.0:1734016384.982022:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.982026:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016384.982030:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016384.982038:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335158400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016384.982047:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.982050:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016384.982052:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.982055:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 649 00000100:00100000:1.0:1734016384.982056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335158144:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016384.982058:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.982061:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016384.982065:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016384.982074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335158528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016384.982082:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.982086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016384.982087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.982090:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 649 00000100:00080000:2.0:1734016384.982093:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.982096:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016384.982101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016384.982108:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335158656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016384.982117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.982119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016384.982121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.982124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 649 00000800:00000100:1.0:1734016384.982126:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016384.982127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.982131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016384.982134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016384.982142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335158784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016384.982148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86de300 x1818247335158144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016384.982149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016384.982152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016384.982153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016384.982157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 649 00000100:00080000:2.0:1734016384.982159:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016384.982162:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016384.982166:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016384.982168:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335158144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016384.982173:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335158912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016384.982178:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982182:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982187:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335158144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016384.982194:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335158272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016384.982214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016384.982221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86dd180 x1818247335158272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016384.982232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335158272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016384.982239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335158272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016384.982269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335158400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016384.982283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016384.982289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86dea00 x1818247335158400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016384.982299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335158400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016384.982306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335158400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016384.982317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335158528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016384.982329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016384.982334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86dce00 x1818247335158528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016384.982351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335158528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016384.982358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335158528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016384.982369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016384.982380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016384.982386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86dc000 x1818247335158656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016384.982396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335158656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016384.982403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016384.982412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016384.982424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016384.982430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86ddc00 x1818247335158784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016384.982440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335158784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016384.982447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016384.982458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016384.982470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016384.982476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016384/real 1734016384] req@ffff8800a86df800 x1818247335158912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016384.982486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335158912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016384.982492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016384.982494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016384.982497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335158912:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016385.749990:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016388.755964:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016389.989852:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.989862:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016389.989908:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.989920:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.989930:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.989938:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016389.989976:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335159040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016389.990011:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.990015:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016389.990017:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.990022:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.990025:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.990028:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016389.990039:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016389.990049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335159168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016389.990059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.990062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016389.990064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.990067:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.990070:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.990073:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016389.990078:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016389.990085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335159296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016389.990095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.990098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016389.990100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.990103:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.990106:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.990109:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016389.990113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016389.990122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335159424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016389.990131:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335159040:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016389.990131:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.990134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016389.990135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.990139:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.990141:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.990145:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016389.990149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016389.990156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335159552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016389.990164:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.990167:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016389.990169:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.990172:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.990176:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.990179:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016389.990183:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016389.990190:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335159680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016389.990198:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016389.990201:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1734016389.990202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016389.990202:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016389.990205:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 654 00000100:00080000:2.0:1734016389.990208:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016389.990211:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016389.990215:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016389.990222:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335159808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016389.990224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86dfb80 x1818247335159040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335159040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990262:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335159040:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016389.990269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335159168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016389.990288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016389.990294:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86ddf80 x1818247335159168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335159168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990313:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335159168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016389.990324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335159296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016389.990338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016389.990344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86dc380 x1818247335159296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335159296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335159296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016389.990390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335159424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016389.990403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016389.990409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86ded80 x1818247335159424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335159424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335159424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016389.990445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335159552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016389.990457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016389.990463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86de680 x1818247335159552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335159552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335159552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016389.990490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335159680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016389.990501:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016389.990508:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86dd500 x1818247335159680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335159680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990528:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335159680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016389.990536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335159808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016389.990549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016389.990554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016389/real 1734016389] req@ffff8800a86df480 x1818247335159808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016389.990564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335159808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016389.990571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016389.990573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016389.990576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335159808:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016391.761896:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016394.767983:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016394.997864:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.997873:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016394.997878:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.997888:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.997898:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016394.997906:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016394.997942:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335159936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016394.997982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.997986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016394.997988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.997992:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.997995:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016394.997998:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016394.998008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016394.998018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335160064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016394.998027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.998030:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016394.998032:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.998035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.998038:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016394.998041:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016394.998045:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016394.998053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335160192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016394.998061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.998064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016394.998066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.998069:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.998072:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016394.998076:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016394.998080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016394.998089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335160320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016394.998097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.998100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016394.998102:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.998105:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.998108:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734016394.998109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335159936:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016394.998111:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016394.998115:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016394.998122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335160448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016394.998130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.998134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016394.998135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.998138:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.998141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016394.998144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016394.998148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016394.998155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335160576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016394.998162:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016394.998166:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016394.998168:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016394.998171:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 659 00000100:00080000:2.0:1734016394.998174:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016394.998177:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734016394.998178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734016394.998180:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016394.998187:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335160704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016394.998199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86dd880 x1818247335159936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335159936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998230:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335159936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016394.998245:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016394.998265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016394.998271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86df100 x1818247335160064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335160064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998290:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998296:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016394.998301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016394.998319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016394.998325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86dc700 x1818247335160192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335160192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016394.998371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016394.998384:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016394.998390:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86dca80 x1818247335160320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998407:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335160320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016394.998425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335160448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016394.998436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016394.998442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86df480 x1818247335160448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335160448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335160448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016394.998469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335160576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016394.998481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016394.998487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86dd500 x1818247335160576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335160576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335160576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016394.998516:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335160704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016394.998528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016394.998534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016394/real 1734016394] req@ffff8800a86de680 x1818247335160704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016394.998544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335160704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016394.998550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016394.998552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016394.998555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335160704:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016397.773991:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016400.005844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.005857:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016400.005863:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.005876:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 664 00000100:00080000:2.0:1734016400.005886:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.005896:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016400.005943:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335160832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016400.005982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.005986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016400.005988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.005993:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 664 00000100:00080000:2.0:1734016400.005996:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.006000:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016400.006012:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016400.006024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335160960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016400.006034:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.006038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016400.006039:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.006043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 664 00000100:00080000:2.0:1734016400.006045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.006049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016400.006054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016400.006063:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335161088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016400.006072:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.006076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016400.006078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.006081:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 664 00000100:00080000:2.0:1734016400.006084:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.006087:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016400.006091:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016400.006102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335161216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016400.006112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.006115:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1734016400.006117:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335160832:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016400.006117:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.006123:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 664 00000100:00080000:2.0:1734016400.006125:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.006129:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016400.006134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016400.006143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335161344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016400.006151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.006154:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016400.006155:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.006159:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 664 00000100:00080000:2.0:1734016400.006162:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.006165:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016400.006169:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016400.006177:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335161472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016400.006185:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016400.006188:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016400.006189:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016400.006193:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 664 00000800:00000100:1.0:1734016400.006194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016400.006196:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016400.006200:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016400.006204:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016400.006211:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335161600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016400.006219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86ded80 x1818247335160832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335160832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335160832:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016400.006269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016400.006290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016400.006297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86dc380 x1818247335160960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335160960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335160960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016400.006328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016400.006342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016400.006348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86ddf80 x1818247335161088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335161088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006366:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016400.006398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016400.006411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016400.006417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86dfb80 x1818247335161216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335161216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016400.006455:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335161344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016400.006467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016400.006473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86df800 x1818247335161344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335161344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335161344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016400.006501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335161472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016400.006514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016400.006520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86ddc00 x1818247335161472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335161472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335161472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016400.006549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335161600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016400.006561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016400.006567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016400/real 1734016400] req@ffff8800a86dc000 x1818247335161600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016400.006577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335161600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016400.006584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016400.006586:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016400.006589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335161600:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016400.779937:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016403.785954:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016405.013829:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016405.013838:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016405.013842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.013853:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.013863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.013871:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016405.013907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335161728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016405.013943:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016405.013947:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016405.013949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.013954:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.013957:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.013960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016405.013970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016405.013980:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335161856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016405.013991:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016405.013994:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016405.014004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.014007:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.014010:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.014014:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016405.014019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016405.014027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335161984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016405.014035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016405.014038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016405.014040:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.014043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.014046:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.014049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016405.014053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016405.014062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335162112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016405.014070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335161728:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016405.014070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016405.014074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016405.014075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.014079:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.014081:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.014084:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016405.014088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016405.014096:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335162240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016405.014104:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016405.014107:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016405.014109:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.014113:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.014115:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.014118:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016405.014122:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016405.014130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335162368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016405.014137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734016405.014138:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016405.014140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016405.014141:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016405.014145:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 669 00000100:00080000:2.0:1734016405.014147:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016405.014150:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016405.014154:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016405.014160:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86dce00 x1818247335161728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016405.014161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335162496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016405.014179:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335161728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014190:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014193:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014198:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335161728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016405.014205:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016405.014225:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016405.014232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86dea00 x1818247335161856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016405.014243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335161856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014251:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016405.014263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016405.014276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016405.014282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86dd180 x1818247335161984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016405.014292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335161984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014299:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335161984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016405.014329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335162112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016405.014341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016405.014347:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86de300 x1818247335162112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016405.014364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335162112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335162112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016405.014382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335162240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016405.014394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016405.014399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86dc000 x1818247335162240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016405.014409:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335162240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335162240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016405.014427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335162368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016405.014438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016405.014444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86ddc00 x1818247335162368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016405.014454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335162368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014461:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014464:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335162368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016405.014473:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335162496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016405.014485:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016405.014491:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016405/real 1734016405] req@ffff8800a86df800 x1818247335162496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016405.014500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335162496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016405.014507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016405.014509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016405.014512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335162496:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016406.791955:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016409.797955:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016410.021836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016410.021845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016410.021849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.021859:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.021868:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.021876:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016410.021913:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335162624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016410.021948:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016410.021952:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016410.021954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.021959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.021962:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.021966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016410.021975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016410.021986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335162752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016410.021995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016410.021999:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016410.022000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.022004:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.022006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.022010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016410.022014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016410.022022:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335162880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016410.022030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016410.022033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016410.022036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.022039:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.022042:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.022045:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016410.022049:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016410.022058:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335163008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016410.022067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016410.022069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335162624:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016410.022070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016410.022071:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.022075:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.022078:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.022081:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016410.022085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016410.022092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335163136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016410.022100:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016410.022103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016410.022104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.022107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.022110:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.022113:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016410.022117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016410.022124:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335163264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016410.022133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734016410.022135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016410.022136:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016410.022137:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016410.022141:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 674 00000100:00080000:2.0:1734016410.022143:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016410.022146:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016410.022150:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016410.022157:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86dfb80 x1818247335162624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016410.022157:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335163392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016410.022177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335162624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022188:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022196:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335162624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016410.022203:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335162752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016410.022222:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016410.022229:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86ddf80 x1818247335162752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016410.022240:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335162752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335162752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016410.022259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335162880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016410.022273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016410.022279:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86dc380 x1818247335162880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016410.022289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335162880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022298:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335162880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016410.022325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016410.022338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016410.022344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86ded80 x1818247335163008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016410.022361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335163008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016410.022378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335163136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016410.022390:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016410.022396:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86de680 x1818247335163136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016410.022405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335163136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335163136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016410.022423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335163264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016410.022434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016410.022440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86dd500 x1818247335163264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016410.022450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335163264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335163264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016410.022468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335163392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016410.022480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016410.022486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016410/real 1734016410] req@ffff8800a86df480 x1818247335163392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016410.022495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335163392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016410.022502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016410.022504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016410.022508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335163392:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016412.803957:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016415.029809:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.029818:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016415.029823:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.029833:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00080000:2.0:1734016415.029843:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.029851:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016415.029887:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335163520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016415.029921:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.029925:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016415.029927:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.029932:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00080000:2.0:1734016415.029935:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.029939:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016415.029949:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016415.029960:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335163648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016415.029970:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.029973:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016415.029975:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.029979:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00080000:2.0:1734016415.029982:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.029985:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016415.029989:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016415.029997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335163776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016415.030006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.030009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016415.030010:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.030014:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00080000:2.0:1734016415.030016:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.030020:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016415.030023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016415.030033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335163904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016415.030038:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335163520:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016415.030042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.030046:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016415.030048:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.030051:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00080000:2.0:1734016415.030054:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.030057:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016415.030061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016415.030069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335164032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016415.030076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.030079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016415.030081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.030084:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00080000:2.0:1734016415.030086:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.030090:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016415.030093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016415.030100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335164160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016415.030105:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016415.030108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016415.030111:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016415.030112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016415.030116:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 679 00000100:00100000:1.0:1734016415.030126:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86dca80 x1818247335163520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016415.030128:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016415.030131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016415.030136:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016415.030143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335164288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016415.030146:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335163520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030156:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030159:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030164:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335163520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016415.030171:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016415.030190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016415.030197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86dc700 x1818247335163648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016415.030209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335163648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030219:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016415.030228:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016415.030241:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016415.030247:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86df100 x1818247335163776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016415.030257:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335163776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030264:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030266:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016415.030292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016415.030305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016415.030311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86dd880 x1818247335163904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016415.030328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335163904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335163904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016415.030346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016415.030357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016415.030363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86df480 x1818247335164032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016415.030372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335164032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016415.030390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016415.030401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016415.030406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86dd500 x1818247335164160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016415.030416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335164160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016415.030487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016415.030505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016415.030513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016415/real 1734016415] req@ffff8800a86de680 x1818247335164288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016415.030524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335164288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016415.030531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016415.030533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016415.030536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164288:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016415.809924:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016418.815970:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016420.037804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016420.037813:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016420.037818:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.037829:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.037838:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.037845:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016420.037880:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335164416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016420.037915:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016420.037920:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016420.037922:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.037926:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.037929:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.037933:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016420.037942:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016420.037953:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335164544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016420.037962:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016420.037965:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016420.037967:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.037971:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.037974:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.037977:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016420.037982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016420.037989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335164672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016420.037998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016420.038000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016420.038002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.038005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.038008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.038011:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016420.038015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016420.038024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335164800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016420.038033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016420.038035:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335164416:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016420.038037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016420.038038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.038042:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.038044:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.038047:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016420.038051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016420.038059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335164928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016420.038068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016420.038071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016420.038072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.038076:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.038078:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.038081:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016420.038085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016420.038092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335165056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016420.038099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734016420.038102:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016420.038102:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016420.038104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016420.038107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 684 00000100:00080000:2.0:1734016420.038110:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016420.038113:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016420.038117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016420.038123:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86ded80 x1818247335164416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016420.038124:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335165184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016420.038143:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335164416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038154:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038158:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038163:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335164416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016420.038170:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335164544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016420.038189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016420.038196:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86dc380 x1818247335164544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016420.038208:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335164544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038215:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335164544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016420.038226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335164672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016420.038239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016420.038245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86ddf80 x1818247335164672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016420.038255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335164672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335164672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016420.038292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335164800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016420.038305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016420.038311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86dfb80 x1818247335164800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016420.038328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335164800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335164800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016420.038345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016420.038404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016420.038413:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86df800 x1818247335164928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016420.038423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335164928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038431:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335164928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016420.038441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016420.038456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016420.038462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86ddc00 x1818247335165056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016420.038472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335165056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016420.038491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016420.038503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016420.038509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016420/real 1734016420] req@ffff8800a86dc000 x1818247335165184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016420.038519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335165184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016420.038526:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016420.038528:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016420.038531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165184:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016421.821962:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016424.828015:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016425.045847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016425.045860:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016425.045866:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.045880:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.045892:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.045902:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016425.045951:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335165312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016425.045990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016425.045994:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016425.045996:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.046001:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.046004:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.046009:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016425.046020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016425.046031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335165440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016425.046041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016425.046044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016425.046046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.046050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.046053:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.046056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016425.046061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016425.046070:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335165568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016425.046079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016425.046083:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016425.046085:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.046089:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.046091:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.046095:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016425.046099:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016425.046109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335165696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016425.046119:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016425.046122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335165312:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016425.046122:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016425.046124:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.046129:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.046131:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.046135:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016425.046139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016425.046148:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335165824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016425.046156:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016425.046159:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016425.046161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.046164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.046167:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.046171:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016425.046176:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016425.046184:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335165952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016425.046192:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734016425.046195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016425.046195:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016425.046197:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016425.046200:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 689 00000100:00080000:2.0:1734016425.046204:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016425.046207:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016425.046211:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016425.046218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86de300 x1818247335165312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016425.046219:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335166080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016425.046239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335165312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046251:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046260:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335165312:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016425.046268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335165440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016425.046339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016425.046347:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86dd180 x1818247335165440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016425.046359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335165440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335165440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016425.046380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335165568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016425.046394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016425.046400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86dea00 x1818247335165568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016425.046411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335165568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335165568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016425.046451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335165696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016425.046464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016425.046470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86dce00 x1818247335165696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016425.046487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335165696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335165696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016425.046506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016425.046518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016425.046524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86dc000 x1818247335165824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016425.046534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335165824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046541:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046543:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046546:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016425.046552:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016425.046564:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016425.046570:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86ddc00 x1818247335165952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016425.046580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335165952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335165952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016425.046599:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016425.046611:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016425.046617:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016425/real 1734016425] req@ffff8800a86df800 x1818247335166080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016425.046626:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335166080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016425.046633:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016425.046635:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016425.046638:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166080:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016427.833996:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016430.053820:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.053833:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016430.053839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016430.053852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.053863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.053872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016430.053921:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335166208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016430.053960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.053965:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016430.053967:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016430.053972:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.053976:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.053979:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016430.053991:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016430.054003:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335166336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016430.054013:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.054017:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016430.054018:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016430.054022:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.054025:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.054029:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016430.054033:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016430.054041:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335166464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016430.054051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.054054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016430.054055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016430.054059:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.054061:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.054065:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016430.054069:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016430.054078:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335166592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016430.054089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.054093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016430.054095:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734016430.054098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335166208:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016430.054099:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.054101:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.054105:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016430.054109:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016430.054118:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335166720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016430.054126:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.054129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016430.054130:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016430.054134:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.054136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.054139:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016430.054143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016430.054151:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335166848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016430.054160:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016430.054163:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016430.054164:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016430.054167:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 694 00000100:00080000:2.0:1734016430.054171:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016430.054175:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016430.054179:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016430.054216:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335166976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016430.054228:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335166336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016430.054254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335166464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016430.054281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335166592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016430.054297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016430.054315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016430.054329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166976:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016430.839948:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016433.060226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86ded80 x1818247335166592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335166592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060297:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335166592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016433.060308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86de680 x1818247335166720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335166720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016433.060362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86dd500 x1818247335166848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335166848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016433.060413:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86df480 x1818247335166976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335166976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335166976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016433.060474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86dfb80 x1818247335166208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335166208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060511:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335166208:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016433.060528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86ddf80 x1818247335166336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335166336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335166336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016433.060577:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016430/real 1734016433] req@ffff8800a86dc380 x1818247335166464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016433.060619:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335166464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016433.060633:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016433.060638:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016433.060643:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335166464:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016433.845992:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016435.061804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.061813:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016435.061818:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.061828:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.061837:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.061845:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016435.061881:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335167104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016435.061915:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.061919:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016435.061921:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.061926:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.061929:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.061932:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016435.061942:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016435.061952:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335167232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016435.061968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.061971:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016435.061973:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.061976:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.061979:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.061982:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016435.061986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016435.061994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335167360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016435.062002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.062005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016435.062007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.062010:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.062013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.062016:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016435.062020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016435.062026:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335167104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016435.062029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335167488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016435.062037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.062040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016435.062042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.062045:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.062048:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.062051:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016435.062056:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016435.062064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335167616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016435.062071:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.062074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016435.062076:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.062079:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.062082:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.062086:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016435.062089:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734016435.062094:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016435.062097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335167744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016435.062140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016435.062143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016435.062144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016435.062148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 699 00000100:00080000:2.0:1734016435.062151:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016435.062154:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062156:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86dd880 x1818247335167104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734016435.062158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016435.062167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335167872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016435.062176:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335167104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062186:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062189:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062194:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335167104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016435.062202:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335167232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016435.062222:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062229:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86df100 x1818247335167232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016435.062240:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335167232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062255:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335167232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016435.062261:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335167360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016435.062275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86dc700 x1818247335167360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016435.062291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335167360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062299:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335167360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016435.062327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335167488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016435.062340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86dca80 x1818247335167488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016435.062356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335167488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335167488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016435.062374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335167616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016435.062390:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86dc380 x1818247335167616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016435.062406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335167616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062418:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335167616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016435.062424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335167744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016435.062435:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86ddf80 x1818247335167744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016435.062450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335167744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335167744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016435.062468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335167872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016435.062479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016435.062485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016435/real 1734016435] req@ffff8800a86dfb80 x1818247335167872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016435.062502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335167872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016435.062509:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016435.062511:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016435.062514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335167872:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016436.851934:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016439.857961:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016440.069822:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016440.069831:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016440.069835:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.069846:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.069856:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.069863:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016440.069899:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335168000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016440.069933:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016440.069937:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016440.069939:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.069944:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.069947:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.069950:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016440.069960:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016440.069971:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335168128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016440.069980:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016440.069984:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016440.069985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.069989:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.069992:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.069996:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016440.070001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016440.070009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335168256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016440.070048:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016440.070051:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016440.070053:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.070057:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.070060:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.070063:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016440.070068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016440.070078:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335168384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016440.070088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016440.070089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335168000:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016440.070091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016440.070092:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.070096:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.070099:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.070102:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016440.070106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016440.070113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335168512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016440.070121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016440.070125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016440.070126:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.070130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.070133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.070136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016440.070140:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016440.070147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335168640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016440.070155:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016440.070155:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016440.070158:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016440.070159:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016440.070163:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 704 00000100:00080000:2.0:1734016440.070165:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016440.070168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016440.070172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016440.070177:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86df480 x1818247335168000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016440.070179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335168768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016440.070196:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335168000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070207:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070210:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070215:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335168000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016440.070222:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335168128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016440.070238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016440.070244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86dd500 x1818247335168128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016440.070255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335168128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335168128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016440.070276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335168256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016440.070290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016440.070296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86de680 x1818247335168256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016440.070306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335168256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335168256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016440.070346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335168384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016440.070358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016440.070363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86ded80 x1818247335168384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016440.070374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335168384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335168384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016440.070391:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335168512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016440.070411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016440.070417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86df800 x1818247335168512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016440.070428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335168512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070440:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335168512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016440.070444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335168640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016440.070456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016440.070461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86ddc00 x1818247335168640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016440.070471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335168640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335168640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016440.070489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335168768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016440.070500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016440.070506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016440/real 1734016440] req@ffff8800a86dc000 x1818247335168768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016440.070522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335168768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016440.070528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016440.070530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016440.070533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335168768:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016442.863850:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016445.077839:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016445.077848:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016445.077853:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016445.077863:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.077872:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.077880:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016445.077917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335168896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016445.077976:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016445.077981:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016445.077983:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016445.077987:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.077991:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.077994:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016445.078004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016445.078015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335169024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016445.078025:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016445.078028:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016445.078030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016445.078033:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.078036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.078039:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016445.078043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016445.078051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335169152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016445.078059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016445.078062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016445.078064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016445.078068:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.078071:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.078074:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016445.078078:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016445.078087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335169280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016445.078096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016445.078099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335168896:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016445.078099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016445.078100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016445.078104:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.078106:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.078110:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016445.078114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016445.078121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335169408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016445.078130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016445.078132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016445.078134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016445.078138:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.078140:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.078143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016445.078147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016445.078154:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335169536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016445.078163:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016445.078166:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016445.078168:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734016445.078169:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016445.078171:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 709 00000100:00080000:2.0:1734016445.078174:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016445.078176:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016445.078180:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016445.078187:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335169664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016445.078190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86dce00 x1818247335168896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335168896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335168896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016445.078235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016445.078251:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016445.078258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86dea00 x1818247335169024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335169024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078277:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016445.078290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016445.078304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016445.078309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86dd180 x1818247335169152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335169152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016445.078357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016445.078369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016445.078375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86de300 x1818247335169280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335169280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016445.078404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335169408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016445.078419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016445.078425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86dc000 x1818247335169408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335169408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335169408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016445.078453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335169536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016445.078464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016445.078469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86ddc00 x1818247335169536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335169536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335169536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016445.078496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335169664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016445.078508:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016445.078513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016445/real 1734016445] req@ffff8800a86df800 x1818247335169664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016445.078530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335169664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016445.078537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016445.078539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016445.078542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335169664:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016445.869987:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016448.875962:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016450.085818:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.085829:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016450.085835:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016450.085896:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 714 00000100:00080000:2.0:1734016450.085909:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.085918:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016450.085957:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335169792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016450.085993:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.085998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016450.086000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016450.086004:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 714 00000100:00080000:2.0:1734016450.086007:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.086012:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016450.086022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016450.086032:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335169920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016450.086042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.086045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016450.086047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016450.086051:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 714 00000100:00080000:2.0:1734016450.086053:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.086056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016450.086061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016450.086069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335170048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016450.086077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.086080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016450.086082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016450.086085:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 714 00000100:00080000:2.0:1734016450.086087:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.086091:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016450.086094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016450.086103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335170176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016450.086112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.086116:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016450.086118:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734016450.086119:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335169792:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016450.086122:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 714 00000100:00080000:2.0:1734016450.086125:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.086128:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016450.086132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016450.086139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335170304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016450.086147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.086150:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016450.086151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016450.086154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 714 00000100:00080000:2.0:1734016450.086157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.086160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016450.086164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016450.086171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335170432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016450.086179:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016450.086181:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016450.086183:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016450.086186:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 714 00000800:00000100:1.0:1734016450.086188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016450.086189:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016450.086193:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016450.086197:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016450.086204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335170560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016450.086209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86ded80 x1818247335169792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086228:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335169792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335169792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016450.086254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016450.086270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016450.086277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86de680 x1818247335169920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086288:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335169920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335169920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016450.086309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016450.086323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016450.086329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86dd500 x1818247335170048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335170048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016450.086377:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016450.086389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016450.086395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86df480 x1818247335170176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335170176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016450.086424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335170304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016450.086440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016450.086446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86dfb80 x1818247335170304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335170304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335170304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016450.086474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335170432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016450.086486:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016450.086491:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86ddf80 x1818247335170432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086501:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335170432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335170432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016450.086519:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335170560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016450.086530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016450.086536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016450/real 1734016450] req@ffff8800a86dc380 x1818247335170560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016450.086552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335170560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016450.086559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016450.086561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016450.086564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335170560:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016451.882005:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016454.888000:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016455.093867:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.093876:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016455.093880:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.093890:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.093899:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.093907:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016455.093943:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335170688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016455.093976:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.093981:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016455.093983:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.093987:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.093990:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.093994:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016455.094003:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016455.094014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335170816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016455.094023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.094026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016455.094028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.094032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.094035:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.094038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016455.094043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016455.094051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335170944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016455.094059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.094062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016455.094063:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.094067:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.094069:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.094072:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016455.094076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016455.094086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335171072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016455.094090:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335170688:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016455.094095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.094098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016455.094099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.094103:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.094105:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.094108:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016455.094112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016455.094120:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335171200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016455.094129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.094132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016455.094134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.094137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.094139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.094143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016455.094146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016455.094153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335171328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016455.094157:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016455.094161:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016455.094164:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016455.094166:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016455.094169:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 719 00000100:00080000:2.0:1734016455.094172:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016455.094175:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86dca80 x1818247335170688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734016455.094179:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016455.094186:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335171456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016455.094197:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335170688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094207:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094215:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335170688:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016455.094223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016455.094239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86dc700 x1818247335170816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016455.094257:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335170816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094269:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016455.094279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016455.094292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86df100 x1818247335170944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016455.094308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335170944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335170944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016455.094347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016455.094359:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86dd880 x1818247335171072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016455.094375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335171072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016455.094393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335171200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016455.094409:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094415:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86dc380 x1818247335171200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016455.094425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335171200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094432:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094435:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094438:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335171200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016455.094443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335171328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016455.094454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094459:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86ddf80 x1818247335171328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016455.094469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335171328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335171328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016455.094487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335171456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016455.094499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016455.094504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016455/real 1734016455] req@ffff8800a86dfb80 x1818247335171456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016455.094521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335171456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016455.094528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016455.094530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016455.094533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335171456:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016457.893937:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016460.101783:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101790:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016460.101794:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101807:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101814:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101819:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016460.101845:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335171584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016460.101865:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101867:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016460.101868:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101871:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101872:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101874:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016460.101880:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016460.101885:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335171712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016460.101890:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101892:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016460.101893:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101895:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101896:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101898:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016460.101900:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016460.101905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335171840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016460.101909:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101912:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016460.101913:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101915:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101916:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101918:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016460.101920:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016460.101926:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335171968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016460.101930:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101932:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016460.101932:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101935:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101936:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101938:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016460.101940:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016460.101943:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335171584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016460.101944:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335172096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016460.101948:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101950:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016460.101951:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101953:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101954:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101955:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016460.101958:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016460.101962:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335172224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016460.101967:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016460.101968:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016460.101969:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016460.101971:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 724 00000100:00080000:2.0:1734016460.101972:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016460.101974:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016460.101976:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016460.101979:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335172352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016460.101988:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102001:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86df480 x1818247335171584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102012:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335171584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102018:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102019:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102022:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335171584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016460.102025:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016460.102034:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102038:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86dd500 x1818247335171712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102044:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335171712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102047:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102050:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102051:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016460.102055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016460.102063:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102066:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86de680 x1818247335171840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102071:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335171840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102075:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102077:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102079:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016460.102081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016460.102087:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102090:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86ded80 x1818247335171968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102095:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335171968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102099:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102100:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102102:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335171968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016460.102118:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016460.102126:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102130:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86df800 x1818247335172096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102136:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335172096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102139:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102140:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102142:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016460.102145:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016460.102151:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102153:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86ddc00 x1818247335172224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102158:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335172224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102162:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102163:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102164:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016460.102167:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016460.102173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016460.102176:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016460/real 1734016460] req@ffff8800a86dc000 x1818247335172352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016460.102186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335172352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016460.102190:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016460.102191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016460.102193:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172352:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016460.899918:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016463.905917:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016465.109762:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016465.109770:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016465.109774:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016465.109783:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.109790:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016465.109796:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016465.109823:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335172480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016465.109850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016465.109853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016465.109855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016465.109859:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.109861:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016465.109864:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016465.109872:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016465.109881:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335172608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016465.109889:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016465.109892:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016465.109893:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016465.109896:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.109898:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016465.109901:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016465.109904:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016465.109911:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335172736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016465.109918:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016465.109921:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016465.109923:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016465.109926:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.109928:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734016465.109929:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335172480:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016465.109931:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016465.109934:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016465.109941:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335172864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016465.109950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016465.109953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016465.109955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016465.109957:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.109959:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016465.109962:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016465.109966:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016465.109972:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335172992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016465.109976:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016465.109979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016465.109982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016465.109983:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016465.109986:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.109989:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734016465.109990:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86de300 x1818247335172480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016465.109992:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016465.109995:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016465.110000:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335172480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016465.110002:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335173120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734016465.110006:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110008:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734016465.110009:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734016465.110011:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335172480:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016465.110011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016465.110013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734016465.110015:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335172608:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734016465.110016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 729 00000100:00080000:2.0:1734016465.110019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016465.110021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734016465.110025:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734016465.110025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016465.110028:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86dd180 x1818247335172608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016465.110031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335173248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016465.110034:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335172608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016465.110038:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110040:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016465.110042:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335172608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016465.110045:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335172736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016465.110053:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016465.110056:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86dea00 x1818247335172736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016465.110061:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335172736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016465.110065:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110067:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016465.110069:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335172736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016465.110083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335172864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016465.110089:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016465.110092:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86dce00 x1818247335172864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016465.110098:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335172864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016465.110102:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110103:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016465.110104:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335172864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016465.110107:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016465.110115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016465.110118:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86dc000 x1818247335172992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016465.110124:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335172992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016465.110127:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110128:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016465.110130:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335172992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016465.110133:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335173120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016465.110139:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016465.110142:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86ddc00 x1818247335173120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016465.110146:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335173120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016465.110150:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110151:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016465.110152:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335173120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016465.110155:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335173248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016465.110161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016465.110164:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016465/real 1734016465] req@ffff8800a86df800 x1818247335173248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016465.110173:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335173248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016465.110176:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016465.110177:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016465.110179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335173248:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016466.911899:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016469.917842:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016470.117803:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117810:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016470.117813:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016470.117820:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117827:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117832:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016470.117859:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335173376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016470.117880:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117882:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016470.117883:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016470.117885:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117887:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117889:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016470.117895:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016470.117901:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335173504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016470.117909:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117912:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016470.117912:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016470.117914:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117915:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117917:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016470.117920:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016470.117924:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335173632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016470.117928:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117929:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016470.117930:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016470.117932:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117933:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117935:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016470.117937:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016470.117942:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335173760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016470.117947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117949:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016470.117949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734016470.117951:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335173376:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016470.117951:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117953:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117955:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016470.117957:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016470.117961:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335173888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016470.117965:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117966:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016470.117967:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016470.117969:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117970:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117972:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016470.117974:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016470.117978:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335174016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016470.117982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016470.117983:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016470.117984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016470.117985:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 734 00000100:00080000:2.0:1734016470.117988:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016470.117990:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016470.117992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734016470.117996:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016470.117996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335174144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016470.118009:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86ded80 x1818247335173376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118019:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335173376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118025:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118027:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118029:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335173376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016470.118033:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335173504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016470.118042:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016470.118045:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86de680 x1818247335173504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118051:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335173504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118055:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118057:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118059:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335173504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016470.118062:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335173632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016470.118069:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016470.118072:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86dd500 x1818247335173632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118077:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335173632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118081:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118083:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118085:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335173632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016470.118099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335173760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016470.118105:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016470.118108:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86df480 x1818247335173760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118113:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335173760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118117:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118118:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118120:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335173760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016470.118123:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335173888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016470.118130:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016470.118133:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86dfb80 x1818247335173888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118138:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335173888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118142:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118143:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118144:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335173888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016470.118147:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016470.118153:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016470.118156:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86ddf80 x1818247335174016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335174016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118165:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118166:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118168:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016470.118170:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016470.118176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016470.118179:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016470/real 1734016470] req@ffff8800a86dc380 x1818247335174144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016470.118188:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335174144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016470.118192:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016470.118193:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016470.118194:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174144:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016472.923926:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016475.125804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.125815:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016475.125820:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.125831:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.125840:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.125849:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016475.125886:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335174272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016475.125920:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.125924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016475.125926:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.125930:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.125933:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.125937:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016475.125947:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016475.125957:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335174400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016475.125966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.125969:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016475.125971:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.125975:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.125978:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.125982:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016475.125986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016475.125995:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335174528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016475.126003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.126006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016475.126007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.126011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.126013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.126016:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016475.126020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016475.126029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335174656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016475.126037:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335174272:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016475.126038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.126041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016475.126043:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.126047:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.126049:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.126052:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016475.126056:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016475.126064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335174784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016475.126072:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.126076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016475.126078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.126081:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.126084:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.126087:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016475.126091:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016475.126098:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335174912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016475.126106:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016475.126108:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1734016475.126110:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016475.126110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016475.126113:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 739 00000100:00080000:2.0:1734016475.126116:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016475.126119:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016475.126122:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016475.126129:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335175040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016475.126131:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86dd880 x1818247335174272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126150:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335174272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126162:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126165:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126170:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335174272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016475.126177:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335174400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016475.126193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016475.126200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86df100 x1818247335174400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335174400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126219:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126222:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335174400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016475.126232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335174528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016475.126246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016475.126252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86dc700 x1818247335174528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335174528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126272:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335174528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016475.126300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335174656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016475.126312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016475.126318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86dca80 x1818247335174656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335174656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335174656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016475.126347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016475.126363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016475.126369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86dc380 x1818247335174784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335174784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016475.126397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016475.126408:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016475.126414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86ddf80 x1818247335174912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335174912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335174912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016475.126475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016475.126490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016475.126497:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016475/real 1734016475] req@ffff8800a86dfb80 x1818247335175040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016475.126514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335175040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016475.126522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016475.126524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016475.126527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175040:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016475.929925:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016478.936006:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016480.133853:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.133864:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016480.133869:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.133880:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00080000:2.0:1734016480.133890:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016480.133899:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016480.133936:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335175168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016480.133972:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.133976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016480.133978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.133983:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00080000:2.0:1734016480.133986:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016480.133989:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016480.133999:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016480.134009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335175296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016480.134019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.134022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016480.134024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.134027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00080000:2.0:1734016480.134030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016480.134033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016480.134037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016480.134045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335175424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016480.134053:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.134056:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016480.134058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.134062:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00080000:2.0:1734016480.134064:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016480.134067:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016480.134071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016480.134081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335175552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016480.134089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.134093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016480.134094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.134098:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00100000:1.0:1734016480.134099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335175168:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016480.134100:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016480.134104:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016480.134108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016480.134115:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335175680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016480.134122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.134125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016480.134127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.134130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00080000:2.0:1734016480.134132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016480.134136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016480.134139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016480.134146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335175808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016480.134155:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016480.134158:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016480.134159:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016480.134162:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 744 00000100:00080000:2.0:1734016480.134165:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734016480.134168:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016480.134168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016480.134172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016480.134179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335175936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016480.134190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86df480 x1818247335175168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335175168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335175168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016480.134234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335175296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016480.134250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016480.134257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86dd500 x1818247335175296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335175296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335175296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016480.134289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335175424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016480.134303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016480.134309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86de680 x1818247335175424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335175424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335175424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016480.134409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335175552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016480.134426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016480.134434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86ded80 x1818247335175552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335175552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335175552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016480.134463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016480.134479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016480.134485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86df800 x1818247335175680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335175680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016480.134513:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016480.134524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016480.134530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86ddc00 x1818247335175808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335175808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016480.134556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016480.134568:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016480.134574:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016480/real 1734016480] req@ffff8800a86dc000 x1818247335175936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016480.134590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335175936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016480.134597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016480.134599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016480.134602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335175936:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016481.942006:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016484.948029:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016485.141908:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.141921:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016485.141925:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.141937:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.141946:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016485.141955:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016485.141994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335176064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016485.142032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.142037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016485.142039:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.142043:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.142046:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016485.142050:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016485.142061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016485.142071:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335176192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016485.142081:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.142084:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016485.142086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.142090:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.142092:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016485.142096:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016485.142100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016485.142107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335176320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016485.142116:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.142119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016485.142120:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.142124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.142126:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734016485.142129:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335176064:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016485.142130:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016485.142134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016485.142144:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335176448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016485.142152:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.142156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016485.142157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.142161:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.142164:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016485.142167:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016485.142170:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016485.142178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335176576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016485.142186:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.142189:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016485.142190:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.142194:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.142196:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016485.142199:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016485.142203:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734016485.142210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016485.142211:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335176704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016485.142218:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016485.142221:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016485.142222:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016485.142226:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 749 00000100:00080000:2.0:1734016485.142230:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016485.142234:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86dce00 x1818247335176064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734016485.142239:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016485.142246:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335176832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016485.142305:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335176064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142321:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335176064:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016485.142345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335176192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016485.142379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86dea00 x1818247335176192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016485.142412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335176192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142425:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335176192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016485.142470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335176320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016485.142502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86dd180 x1818247335176320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016485.142533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335176320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335176320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016485.142568:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335176448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016485.142596:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142609:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86de300 x1818247335176448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016485.142626:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335176448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142638:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142641:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142647:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335176448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016485.142657:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335176576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016485.142737:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142755:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86dc000 x1818247335176576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016485.142774:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335176576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142787:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142791:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142796:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335176576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016485.142806:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335176704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016485.142838:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142850:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86ddc00 x1818247335176704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016485.142868:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335176704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142880:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142884:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142889:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335176704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016485.142898:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335176832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016485.142923:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016485.142934:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016485/real 1734016485] req@ffff8800a86df800 x1818247335176832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016485.142961:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335176832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016485.142972:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016485.142975:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016485.142980:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335176832:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016487.953830:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016488.955994:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016488.956445:0:1971:0:(client.c:1722:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8800a98b3100 x1818247335176960/t0(0) o103->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:17/18 lens 384/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 00010000:00010000:2.0:1734016489.957822:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016490.149797:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149805:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016490.149808:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149815:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149821:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016490.149826:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016490.149852:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335177088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016490.149874:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149877:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016490.149877:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149880:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149882:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016490.149884:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016490.149890:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016490.149895:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335177216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016490.149904:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149906:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016490.149907:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149909:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149910:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016490.149912:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016490.149914:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016490.149918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335177344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016490.149923:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016490.149925:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149927:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149928:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016490.149930:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016490.149932:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016490.149938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335177472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016490.149942:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149944:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016490.149945:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149947:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149949:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734016490.149950:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335177088:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016490.149951:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016490.149953:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016490.149957:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335177600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016490.149961:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149962:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016490.149963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149965:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149966:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016490.149968:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016490.149970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016490.149974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335177728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016490.149978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016490.149979:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016490.149980:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016490.149981:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 754 00000100:00080000:2.0:1734016490.149983:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016490.149984:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016490.149986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016490.149990:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335177856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016490.149995:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150008:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86ded80 x1818247335177088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150019:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335177088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150025:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150027:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150030:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335177088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016490.150034:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335177216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016490.150043:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150046:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86de680 x1818247335177216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150052:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335177216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150056:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150058:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150059:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335177216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016490.150063:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335177344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016490.150070:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150073:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86dd500 x1818247335177344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150078:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335177344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150082:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150083:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150085:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335177344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016490.150099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335177472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016490.150106:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150109:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86df480 x1818247335177472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150114:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335177472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150118:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150119:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150121:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335177472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016490.150124:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335177600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016490.150132:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86dfb80 x1818247335177600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150139:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335177600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150143:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150144:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150146:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335177600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016490.150148:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335177728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016490.150154:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150157:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86ddf80 x1818247335177728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150189:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335177728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150194:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150195:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335177728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016490.150198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335177856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016490.150207:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016490.150210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016490/real 1734016490] req@ffff8800a86dc380 x1818247335177856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016490.150219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335177856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016490.150223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016490.150224:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016490.150226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335177856:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016490.959888:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016492.963832:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016493.965793:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016495.157751:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157758:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016495.157762:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157772:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157780:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016495.157787:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016495.157823:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335177984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016495.157848:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157850:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016495.157851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157854:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157856:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016495.157858:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016495.157866:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016495.157872:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335178112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016495.157877:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157879:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016495.157880:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157883:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157885:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016495.157887:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016495.157890:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016495.157896:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335178240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016495.157901:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157903:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016495.157904:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157906:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157907:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016495.157909:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016495.157911:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016495.157918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335178368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016495.157922:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016495.157925:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157927:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157928:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016495.157930:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016495.157932:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016495.157938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335178496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016495.157947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157949:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016495.157950:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157953:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157954:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734016495.157955:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335177984:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016495.157956:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016495.157959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016495.157964:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335178624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016495.157968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016495.157969:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016495.157970:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016495.157972:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 759 00000100:00080000:2.0:1734016495.157973:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016495.157975:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016495.157977:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016495.157982:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335178752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016495.158067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335178112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016495.158116:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335178240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016495.158135:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335178368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016495.158158:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335178496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016495.158174:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335178624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016495.158188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335178752:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016495.970044:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016496.971999:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1734016498.164083:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734016498.164140:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734016498.164160:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016498.164208:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016498.164220:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016498.164230:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016498.164240:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016498.164250:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016498.164260:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00010000:00010000:2.0:1734016498.975989:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016499.977976:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016500.165865:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.165877:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016500.165882:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.165893:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.165902:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.165911:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016500.165948:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335178880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016500.165983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.165987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016500.165989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.166035:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.166039:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.166045:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016500.166058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016500.166076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335179008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016500.166091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.166096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016500.166099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.166104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.166108:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.166114:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016500.166121:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016500.166137:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335179136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016500.166151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.166171:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016500.166173:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.166178:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.166183:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.166188:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016500.166194:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016500.166207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335179264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016500.166220:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.166224:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016500.166227:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.166232:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.166236:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.166242:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016500.166249:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016500.166262:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335179392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016500.166273:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.166276:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016500.166278:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.166281:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.166285:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.166288:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016500.166293:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016500.166301:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335179520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016500.166308:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016500.166311:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016500.166312:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016500.166316:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 764 00000100:00080000:2.0:1734016500.166319:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016500.166322:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016500.166326:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016500.166333:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335179648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016501.981866:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016502.983861:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016504.988257:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016505.173859:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.173869:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016505.173874:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.173884:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.173893:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.173900:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016505.173968:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335179776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016505.173994:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.173997:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016505.173998:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.174001:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.174003:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.174006:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016505.174015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016505.174022:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335179904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016505.174027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.174029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016505.174030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.174032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.174034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.174036:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016505.174039:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016505.174045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335180032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016505.174050:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.174052:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016505.174053:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.174055:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.174057:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.174059:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016505.174061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016505.174068:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335180160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016505.174073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.174075:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016505.174076:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.174078:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.174079:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.174082:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016505.174084:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016505.174090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335180288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016505.174095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.174097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016505.174098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.174101:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.174103:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.174105:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016505.174108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016505.174113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335180416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016505.174117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016505.174119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016505.174119:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016505.174121:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 769 00000100:00080000:2.0:1734016505.174123:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016505.174125:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016505.174127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016505.174132:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335180544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016505.989926:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016507.993913:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016508.996216:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016510.181811:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.181853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016510.181859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.181872:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.181885:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.181895:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016510.181943:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335180672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016510.181983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.181990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016510.181993:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.181999:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.182005:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.182011:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016510.182026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016510.182046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335180800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016510.182062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.182066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016510.182068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.182074:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.182078:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.182083:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016510.182090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016510.182103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335180928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016510.182114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.182119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016510.182121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.182126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.182132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.182138:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016510.182144:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016510.182159:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335181056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016510.182171:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.182176:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016510.182179:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.182185:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.182189:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.182194:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016510.182200:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016510.182212:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335181184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016510.182225:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.182230:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016510.182233:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.182238:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.182242:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.182246:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016510.182252:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016510.182263:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335181312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016510.182276:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016510.182282:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016510.182285:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016510.182290:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 774 00000100:00080000:2.0:1734016510.182296:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016510.182301:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016510.182307:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016510.182320:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335181440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016511.000036:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016512.001952:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016514.005991:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016515.007990:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016515.189843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.189852:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016515.189856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.189867:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.189876:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.189884:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016515.189923:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335181568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016515.189957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.189962:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016515.189964:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.189968:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.189972:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.189976:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016515.189985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016515.189996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335181696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016515.190006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.190009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016515.190010:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.190014:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.190016:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.190019:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016515.190024:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016515.190032:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335181824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016515.190041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.190044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016515.190046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.190049:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.190051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.190055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016515.190059:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016515.190067:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335181952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016515.190077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.190080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016515.190082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.190085:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.190088:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.190091:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016515.190095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016515.190104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335182080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016515.190111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.190114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016515.190116:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.190119:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.190121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.190124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016515.190128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016515.190136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335182208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016515.190143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016515.190146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016515.190147:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016515.190151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 779 00000100:00080000:2.0:1734016515.190153:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016515.190156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016515.190160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016515.190168:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335182336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016517.011987:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016518.013993:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016520.017987:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016520.198154:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335182464:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016520.198238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198264:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59cb800 x1818247335182464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335182464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335182464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016520.198313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335182592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016520.198330:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59c8000 x1818247335182592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335182592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335182592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016520.198391:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335182720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016520.198405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59c9c00 x1818247335182720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335182720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198431:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335182720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016520.198442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335182848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016520.198453:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198459:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59cbb80 x1818247335182848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335182848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335182848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016520.198487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335182976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016520.198500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59c9880 x1818247335182976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335182976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198530:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335182976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016520.198536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335183104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016520.198550:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59c9500 x1818247335183104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335183104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198573:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335183104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016520.198583:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335183232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016520.198596:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016520.198603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016520/real 1734016520] req@ffff8800b59ca300 x1818247335183232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016520.198613:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335183232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016520.198627:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016520.198630:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016520.198633:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335183232:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016521.019936:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016523.023980:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016524.025959:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016525.206092:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335183360:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016525.206175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59ca680 x1818247335183360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335183360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335183360:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016525.206250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335183488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016525.206267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59c8e00 x1818247335183488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335183488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335183488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016525.206306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335183616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016525.206322:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206328:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59cb100 x1818247335183616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335183616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206346:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206349:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206352:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335183616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016525.206381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335183744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016525.206393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59cb480 x1818247335183744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335183744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335183744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016525.206429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335183872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016525.206445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59c9f80 x1818247335183872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335183872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335183872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016525.206479:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016525.206490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59c8a80 x1818247335184000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335184000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016525.206522:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016525.206535:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016525.206541:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016525/real 1734016525] req@ffff8800b59caa00 x1818247335184128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016525.206557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335184128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016525.206610:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016525.206613:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016525.206617:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184128:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016526.029972:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016527.031976:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016529.035986:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016530.037990:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016530.214089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335184256:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016530.214175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59c8700 x1818247335184256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335184256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335184256:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016530.214249:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335184384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016530.214267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59caa00 x1818247335184384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214294:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335184384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335184384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016530.214314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335184512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016530.214328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59c8a80 x1818247335184512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335184512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335184512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016530.214389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335184640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016530.214403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59c9f80 x1818247335184640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335184640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335184640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016530.214441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016530.214454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59cb480 x1818247335184768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335184768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016530.214543:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016530.214560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59cb100 x1818247335184896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335184896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335184896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016530.214598:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016530.214615:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016530.214622:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016530/real 1734016530] req@ffff8800b59c8e00 x1818247335185024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016530.214633:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335185024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016530.214641:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016530.214643:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016530.214646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185024:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016532.041956:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016533.043938:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016535.047972:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016535.222077:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335185152:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016535.222146:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222168:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59ca680 x1818247335185152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222188:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335185152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222199:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222202:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335185152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016535.222214:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335185280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016535.222230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222237:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59ca300 x1818247335185280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335185280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222262:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335185280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016535.222268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335185408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016535.222282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59c9500 x1818247335185408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335185408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335185408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016535.222337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335185536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016535.222349:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59c9880 x1818247335185536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335185536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335185536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016535.222385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016535.222444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59cbb80 x1818247335185664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335185664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016535.222483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016535.222498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59c9c00 x1818247335185792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335185792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016535.222532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016535.222547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016535.222553:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016535/real 1734016535] req@ffff8800b59c8000 x1818247335185920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016535.222563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335185920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016535.222570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016535.222572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016535.222576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335185920:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016536.049976:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016538.053958:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016539.055931:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016540.230057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335186048:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016540.230126:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59cb800 x1818247335186048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230167:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335186048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230177:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230180:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230184:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335186048:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016540.230191:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335186176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016540.230208:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230214:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59c9180 x1818247335186176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335186176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335186176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016540.230246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335186304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016540.230259:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59cad80 x1818247335186304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230275:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335186304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335186304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016540.230345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335186432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016540.230361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59c8000 x1818247335186432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335186432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335186432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016540.230399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335186560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016540.230411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59c9c00 x1818247335186560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335186560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230435:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230440:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335186560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016540.230445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335186688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016540.230457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59cbb80 x1818247335186688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335186688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230480:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335186688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016540.230490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335186816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016540.230505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016540.230511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016540/real 1734016540] req@ffff8800b59c9880 x1818247335186816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016540.230521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335186816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016540.230528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016540.230530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016540.230533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335186816:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016541.059964:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016542.062004:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016544.065975:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016545.067971:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016545.238080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335186944:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016545.238164:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59c9500 x1818247335186944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335186944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335186944:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016545.238272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016545.238293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59ca300 x1818247335187072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335187072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016545.238356:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016545.238369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59ca680 x1818247335187200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335187200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016545.238405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016545.238416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59c8e00 x1818247335187328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335187328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016545.238458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335187456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016545.238471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59cb100 x1818247335187456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335187456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335187456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016545.238505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335187584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016545.238518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59cb480 x1818247335187584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335187584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238543:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238546:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335187584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016545.238551:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335187712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016545.238563:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016545.238569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016545/real 1734016545] req@ffff8800b59c9f80 x1818247335187712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016545.238579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335187712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016545.238586:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016545.238588:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016545.238591:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335187712:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016547.071959:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016548.073979:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016550.078004:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016550.246098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335187840:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016550.246217:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59c8a80 x1818247335187840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246265:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335187840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335187840:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016550.246294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016550.246317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59caa00 x1818247335187968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335187968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335187968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016550.246355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335188096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016550.246369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59c8700 x1818247335188096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335188096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335188096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016550.246426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335188224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016550.246439:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59c9f80 x1818247335188224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335188224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335188224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016550.246482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335188352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016550.246494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59cb480 x1818247335188352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335188352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335188352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016550.246529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335188480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016550.246541:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59cb100 x1818247335188480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335188480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335188480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016550.246577:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335188608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016550.246589:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016550.246595:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016550/real 1734016550] req@ffff8800b59c8e00 x1818247335188608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016550.246605:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335188608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016550.246612:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016550.246614:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016550.246618:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335188608:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016551.079997:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016553.083958:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016554.085864:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016555.253908:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.253917:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016555.253921:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.253931:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.253942:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.253950:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016555.253987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335188736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016555.254025:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.254032:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016555.254034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.254039:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.254042:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.254045:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016555.254098:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016555.254114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335188864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016555.254125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.254129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016555.254130:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.254134:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.254137:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.254141:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016555.254147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016555.254155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335188992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016555.254165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.254169:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016555.254170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.254174:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.254176:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.254179:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016555.254184:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016555.254192:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335189120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016555.254201:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.254204:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016555.254205:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.254209:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.254211:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.254215:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016555.254218:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016555.254226:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335189248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016555.254234:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.254236:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016555.254238:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.254241:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.254244:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.254248:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016555.254252:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016555.254259:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335189376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016555.254267:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016555.254270:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016555.254272:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016555.254276:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 819 00000100:00080000:2.0:1734016555.254279:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016555.254282:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016555.254286:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016555.254293:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335189504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016556.089993:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016557.091997:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016559.095960:0:5236:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016560.098009:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734016560.261830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.261839:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016560.261844:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.261854:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.261864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.261872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016560.261908:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335189632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016560.261942:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.261947:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016560.261949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.261953:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.261956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.261960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016560.261995:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016560.262008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335189760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016560.262019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.262022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016560.262024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.262027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.262030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.262034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016560.262039:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016560.262047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335189888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016560.262055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.262059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016560.262060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.262064:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.262066:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.262070:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016560.262074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016560.262083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335190016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016560.262092:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.262095:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016560.262098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.262102:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.262104:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.262107:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016560.262112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016560.262120:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335190144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016560.262127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.262130:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016560.262132:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.262135:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.262137:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.262140:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016560.262144:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016560.262151:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335190272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016560.262159:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016560.262162:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016560.262163:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016560.262166:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 824 00000100:00080000:2.0:1734016560.262169:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016560.262172:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016560.262176:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016560.262183:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335190400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016563.268139:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86de300 x1818247335190016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268179:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335190016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268197:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268203:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268210:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016563.268225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86dc000 x1818247335190144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335190144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335190144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016563.268282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86ddc00 x1818247335190272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268316:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335190272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268334:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335190272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016563.268360:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86df800 x1818247335190400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335190400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335190400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016563.268417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86dce00 x1818247335189632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335189632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268456:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335189632:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016563.268473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86dea00 x1818247335189760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335189760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335189760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016563.268525:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016560/real 1734016563] req@ffff8800a86dd180 x1818247335189888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016563.268542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335189888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016563.268555:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016563.268560:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016563.268565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335189888:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734016565.269824:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.269834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016565.269838:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.269849:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00080000:2.0:1734016565.269858:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.269866:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016565.269933:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335190528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016565.269967:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.269971:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016565.269973:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.269978:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00080000:2.0:1734016565.269981:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.269985:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016565.269994:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016565.270005:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335190656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016565.270015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.270018:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016565.270020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.270023:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00080000:2.0:1734016565.270026:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.270029:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016565.270034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016565.270042:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335190784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016565.270051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.270054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016565.270056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.270060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00080000:2.0:1734016565.270062:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.270066:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016565.270070:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016565.270079:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335190912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016565.270087:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.270090:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016565.270092:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.270095:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00080000:2.0:1734016565.270097:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.270101:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016565.270105:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016565.270112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335191040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016565.270122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.270125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016565.270127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.270131:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00100000:1.0:1734016565.270133:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335190528:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016565.270133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.270137:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016565.270141:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016565.270149:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335191168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016565.270156:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016565.270160:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016565.270161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016565.270164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 829 00000100:00080000:2.0:1734016565.270167:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016565.270170:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016565.270174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016565.270181:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335191296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016565.270234:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86ded80 x1818247335190528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335190528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335190528:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016565.270314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016565.270332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86de680 x1818247335190656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335190656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016565.270370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016565.270385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86dd500 x1818247335190784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335190784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016565.270422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016565.270435:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86df480 x1818247335190912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335190912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335190912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016565.270495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016565.270509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86dd880 x1818247335191040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335191040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016565.270546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016565.270559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86df100 x1818247335191168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335191168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270584:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270587:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016565.270593:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016565.270605:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016565.270611:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016565/real 1734016565] req@ffff8800a86dc700 x1818247335191296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016565.270620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335191296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016565.270627:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016565.270630:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016565.270632:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191296:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734016570.277825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.277835:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016570.277839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.277849:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.277859:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.277867:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016570.277903:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335191424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016570.277944:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.277948:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016570.277950:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.277954:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.277957:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.277961:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016570.277970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016570.277980:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335191552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016570.277989:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.277992:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016570.277994:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.277998:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.278000:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.278004:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016570.278008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016570.278016:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335191680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016570.278024:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.278027:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016570.278029:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.278032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.278035:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.278039:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016570.278043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016570.278053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335191808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016570.278061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.278064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016570.278066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.278069:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.278071:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.278075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016570.278079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016570.278086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335191936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016570.278096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.278099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016570.278101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.278105:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.278107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.278110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016570.278114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734016570.278115:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335191424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016570.278122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335192064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016570.278130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016570.278133:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016570.278135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016570.278138:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 834 00000100:00080000:2.0:1734016570.278141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016570.278144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016570.278148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016570.278155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335192192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016570.278216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86dc380 x1818247335191424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335191424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335191424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016570.278299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335191552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016570.278317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278323:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86ddf80 x1818247335191552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335191552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335191552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016570.278355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335191680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016570.278370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86dfb80 x1818247335191680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335191680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335191680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016570.278407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335191808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016570.278420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86dca80 x1818247335191808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335191808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278449:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335191808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016570.278480:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016570.278494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86dc700 x1818247335191936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335191936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335191936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016570.278529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016570.278546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86df100 x1818247335192064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335192064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016570.278582:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016570.278594:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016570.278600:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016570/real 1734016570] req@ffff8800a86dd880 x1818247335192192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016570.278610:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335192192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016570.278617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016570.278620:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016570.278622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192192:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734016575.285825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.285837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016575.285841:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.285852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.285862:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.285870:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016575.285907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335192320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016575.285942:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.285946:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016575.285948:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.285953:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.285956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.285959:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016575.285969:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016575.285980:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335192448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016575.285990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.285993:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016575.285995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.285998:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.286001:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.286004:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016575.286008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016575.286016:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335192576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016575.286024:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.286027:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016575.286029:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.286033:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.286035:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.286038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016575.286042:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016575.286051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ded80 x1818247335192704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016575.286061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.286064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016575.286066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.286069:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.286072:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.286076:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016575.286081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734016575.286083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335192320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016575.286089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335192832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016575.286097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.286099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016575.286101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.286104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.286107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.286110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016575.286114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016575.286121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335192960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016575.286129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016575.286132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016575.286133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016575.286136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 839 00000100:00080000:2.0:1734016575.286139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016575.286142:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016575.286146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016575.286152:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335193088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734016575.286177:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86df480 x1818247335192320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335192320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335192320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016575.286266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335192448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016575.286284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86dd500 x1818247335192448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335192448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335192448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016575.286322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335192576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016575.286337:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86de680 x1818247335192576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335192576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286360:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286364:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286367:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335192576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016575.286398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335192704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016575.286412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86ded80 x1818247335192704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ded80 x1818247335192704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286438:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335192704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016575.286447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016575.286460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86dd180 x1818247335192832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335192832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016575.286496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016575.286512:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86dea00 x1818247335192960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286529:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335192960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335192960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016575.286547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016575.286560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016575.286566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016575/real 1734016575] req@ffff8800a86dce00 x1818247335193088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016575.286576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335193088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016575.286583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016575.286585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016575.286588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193088:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734016580.293859:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.293872:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734016580.293878:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.293891:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.293904:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016580.293914:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734016580.293961:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335193216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016580.294000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.294004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016580.294006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.294011:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.294014:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016580.294018:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016580.294029:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016580.294041:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335193344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016580.294050:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.294054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016580.294056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.294060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.294063:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016580.294066:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016580.294072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734016580.294080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335193472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016580.294089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.294093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016580.294094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.294098:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.294100:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016580.294103:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016580.294108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016580.294118:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335193600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016580.294126:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335193216:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734016580.294127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.294130:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734016580.294132:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.294136:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.294139:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016580.294142:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016580.294147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734016580.294156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335193728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016580.294165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.294168:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734016580.294170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.294174:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.294177:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734016580.294181:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016580.294185:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016580.294193:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335193856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016580.294201:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734016580.294204:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734016580.294205:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734016580.294208:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 844 00000100:00080000:2.0:1734016580.294211:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734016580.294213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734016580.294215:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734016580.294219:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734016580.294226:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335193984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734016580.294239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86df800 x1818247335193216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335193216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294283:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335193216:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016580.294296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335193344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016580.294313:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016580.294319:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86ddc00 x1818247335193344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335193344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335193344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016580.294350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335193472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016580.294364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016580.294370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86dc000 x1818247335193472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335193472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335193472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016580.294422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335193600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016580.294436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016580.294443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86de300 x1818247335193600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335193600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294461:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335193600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016580.294471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016580.294483:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016580.294489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86dce00 x1818247335193728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335193728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016580.294516:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016580.294532:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016580.294538:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86dea00 x1818247335193856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335193856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016580.294566:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016580.294578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016580.294584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016580/real 1734016580] req@ffff8800a86dd180 x1818247335193984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016580.294594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335193984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016580.294601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016580.294603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016580.294606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335193984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016585.302185:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335194112:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016585.302277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59ca680 x1818247335194112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335194112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335194112:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016585.302402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335194240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016585.302428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59ca300 x1818247335194240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302458:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335194240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335194240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016585.302489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335194368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016585.302517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59c9500 x1818247335194368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335194368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335194368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016585.302616:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335194496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016585.302638:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302646:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59c9880 x1818247335194496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302657:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335194496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302664:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302666:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302669:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335194496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016585.302674:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335194624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016585.302688:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302741:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59cbb80 x1818247335194624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302751:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335194624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302759:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302761:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302765:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335194624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016585.302788:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335194752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016585.302809:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302815:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59c9c00 x1818247335194752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302826:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335194752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302834:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302836:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302839:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335194752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016585.302844:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335194880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016585.302856:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016585.302862:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016585/real 1734016585] req@ffff8800b59c8000 x1818247335194880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016585.302872:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335194880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016585.302880:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016585.302882:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016585.302885:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335194880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016590.310026:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335195008:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016590.310116:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310143:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59cad80 x1818247335195008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310180:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335195008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310196:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310200:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335195008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016590.310240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335195136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016590.310266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59c9180 x1818247335195136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335195136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310317:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335195136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016590.310326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335195264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016590.310354:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59cb800 x1818247335195264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335195264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335195264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016590.310412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335195392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016590.310432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59c8000 x1818247335195392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335195392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335195392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016590.310514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335195520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016590.310532:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310541:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59c9c00 x1818247335195520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335195520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335195520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016590.310586:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335195648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016590.310602:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310609:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59cbb80 x1818247335195648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335195648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310627:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310629:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310632:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335195648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016590.310637:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335195776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016590.310649:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016590.310655:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016590/real 1734016590] req@ffff8800b59c9880 x1818247335195776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016590.310665:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335195776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016590.310672:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016590.310674:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016590.310676:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335195776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016595.318100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335195904:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016595.318195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59c9500 x1818247335195904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335195904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318269:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335195904:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016595.318282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016595.318301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59ca300 x1818247335196032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335196032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016595.318339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016595.318353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59ca680 x1818247335196160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335196160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016595.318429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016595.318447:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59c8e00 x1818247335196288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335196288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016595.318514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335196416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016595.318529:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59cb100 x1818247335196416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335196416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318554:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335196416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016595.318565:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335196544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016595.318581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59cb480 x1818247335196544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318598:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335196544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318605:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318608:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318611:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335196544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016595.318616:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335196672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016595.318628:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016595.318634:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016595/real 1734016595] req@ffff8800b59c9f80 x1818247335196672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016595.318644:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335196672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016595.318650:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016595.318653:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016595.318656:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335196672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016600.326113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335196800:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016600.326209:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59c8700 x1818247335196800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335196800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326283:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335196800:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016600.326345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016600.326368:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59caa00 x1818247335196928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335196928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326404:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335196928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016600.326410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016600.326425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59c8a80 x1818247335197056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335197056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016600.326464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016600.326477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59c9f80 x1818247335197184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335197184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016600.326539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335197312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016600.326552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59cb480 x1818247335197312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335197312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335197312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016600.326589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335197440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016600.326604:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326611:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59cb100 x1818247335197440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326622:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335197440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326629:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326631:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326634:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335197440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016600.326640:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335197568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016600.326652:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016600.326658:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016600/real 1734016600] req@ffff8800b59c8e00 x1818247335197568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016600.326668:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335197568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016600.326674:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016600.326677:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016600.326680:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335197568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016605.334072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335197696:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016605.334158:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59ca680 x1818247335197696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334241:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335197696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335197696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016605.334269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016605.334290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59ca300 x1818247335197824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334309:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335197824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016605.334329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016605.334344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59c9500 x1818247335197952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335197952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335197952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016605.334405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016605.334419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59c9880 x1818247335198080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335198080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016605.334453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335198208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016605.334465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59cbb80 x1818247335198208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335198208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335198208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016605.334499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335198336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016605.334514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59c9c00 x1818247335198336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335198336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335198336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016605.334547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335198464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016605.334560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016605.334566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016605/real 1734016605] req@ffff8800b59c8000 x1818247335198464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016605.334575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335198464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016605.334582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016605.334584:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016605.334587:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335198464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016610.342076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335198592:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016610.342214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59cb800 x1818247335198592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335198592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335198592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016610.342302:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016610.342322:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59c9180 x1818247335198720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342340:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335198720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016610.342359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016610.342375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59cad80 x1818247335198848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335198848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016610.342437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016610.342451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59c8000 x1818247335198976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335198976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335198976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016610.342487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335199104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016610.342500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59c9c00 x1818247335199104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335199104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335199104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016610.342535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335199232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016610.342551:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342557:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59cbb80 x1818247335199232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335199232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335199232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016610.342585:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335199360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016610.342598:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016610.342604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016610/real 1734016610] req@ffff8800b59c9880 x1818247335199360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016610.342614:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335199360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016610.342621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016610.342623:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016610.342626:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335199360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016615.350129:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335199488:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016615.350227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350255:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59c9500 x1818247335199488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335199488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350299:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335199488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016615.350315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335199616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016615.350334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59ca300 x1818247335199616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335199616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335199616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016615.350373:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335199744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016615.350388:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350394:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59ca680 x1818247335199744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350404:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335199744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335199744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016615.350452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335199872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016615.350467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59c8e00 x1818247335199872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335199872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335199872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016615.350502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335200000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016615.350515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59cb100 x1818247335200000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335200000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335200000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016615.350550:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335200128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016615.350566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59cb480 x1818247335200128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335200128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350590:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335200128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016615.350601:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335200256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016615.350614:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016615.350619:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016615/real 1734016615] req@ffff8800b59c9f80 x1818247335200256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016615.350629:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335200256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016615.350636:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016615.350638:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016615.350641:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335200256:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016618.213970:0:5236:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 13 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1734016618.214013:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978900/0xd17286ca2ee5074e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xc:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc55993f56 expref: -99 pid: 10406 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1734016618.214046:0:5236:0:(vvp_object.c:134:vvp_conf_set()) [0x200000401:0xc:0x0]: losing layout lock 00010000:00010000:2.0:1734016618.214064:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130979680/0xd17286ca2ee507b0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc55994052 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1734016618.214075:0:5236:0:(vvp_object.c:134:vvp_conf_set()) [0x200000401:0xf:0x0]: losing layout lock 00010000:00010000:2.0:1734016618.214084:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff880130978480/0xd17286ca2ee507a2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x979ba2bc55994021 expref: -99 pid: 10407 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1734016618.214095:0:5236:0:(vvp_object.c:134:vvp_conf_set()) [0x200000401:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1734016618.214101:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a2400/0xd17286ca2ee50763 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xc:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x979ba2bc55993f95 expref: -99 pid: 10407 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1734016618.214122:0:5236:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8800b588c800 lock: ffff8800a87a0480/0xd17286ca2ee4986d lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x7:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x979ba2bc5598ae55 expref: -99 pid: 9662 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1734016618.214160:0:5236:0:(mdc_request.c:909:mdc_close()) @@@ matched open req@ffff8800a8749f80 x1818247332996736/t4294968044(4294968044) o101->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1734016032 ref 1 fl Complete:RPQU/204/ffffffff rc 0/-1 job:'fsx.0' uid:0 gid:0 00000020:00100000:2.0:1734016618.214195:0:5236:0:(genops.c:2298:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8800b588c800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1734016618.214202:0:5236:0:(client.c:782:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8800a86ded80 x1818247335200512/t0(0) o35->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734016618.214226:0:5236:0:(client.c:1722:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8800a86ded80 x1818247335200512/t0(0) o35->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 00000100:00100000:2.0:1734016618.214236:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016619.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016619.215960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016620.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:1.0:1734016620.358071:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335200768:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016620.358167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358195:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59c8a80 x1818247335200768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335200768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335200768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016620.358257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335200896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016620.358276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59caa00 x1818247335200896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335200896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335200896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016620.358338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335201024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016620.358353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59c8700 x1818247335201024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335201024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335201024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016620.358390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335201152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016620.358403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59c9f80 x1818247335201152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335201152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335201152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016620.358437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335201280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016620.358451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59cb480 x1818247335201280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335201280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335201280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016620.358487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335201408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016620.358503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59cb100 x1818247335201408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335201408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335201408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016620.358538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335201536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016620.358553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016620.358559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016620/real 1734016620] req@ffff8800b59c8e00 x1818247335201536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016620.358568:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335201536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016620.358575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016620.358578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016620.358581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335201536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016621.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016622.221974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016624.225984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016625.366094:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335201664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016625.366220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335201792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016625.366246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335201920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016625.366275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016625.366292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335202176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016625.366310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335202304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016625.366325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335202432:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016626.229986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016628.234000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1734016628.372032:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734016628.372085:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734016628.372108:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016628.372152:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016628.372162:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016628.372170:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016628.372177:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016628.372184:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016628.372190:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00010000:00010000:2.0:1734016630.237955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016630.374043:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335202560:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016630.374110:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374131:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59c9500 x1818247335202560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374149:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335202560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374160:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374163:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374168:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335202560:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016630.374175:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016630.374191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59ca300 x1818247335202688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335202688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374217:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374219:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374223:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016630.374229:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016630.374245:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59ca680 x1818247335202816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335202816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374269:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016630.374299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016630.374311:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59c8000 x1818247335202944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335202944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335202944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016630.374346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016630.374358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59c9c00 x1818247335203072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335203072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016630.374393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016630.374404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59cbb80 x1818247335203200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335203200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016630.374437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016630.374448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016630.374454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016630/real 1734016630] req@ffff8800b59c9880 x1818247335203328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016630.374464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335203328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016630.374471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016630.374473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016630.374476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203328:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016632.241944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016634.245957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016635.382057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335203456:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016635.382123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59cad80 x1818247335203456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382164:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335203456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382174:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382178:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382182:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335203456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016635.382189:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335203584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016635.382206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59c9180 x1818247335203584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335203584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382234:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382237:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335203584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016635.382246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335203712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016635.382260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59cb800 x1818247335203712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335203712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335203712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016635.382316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335203840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016635.382328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59c9880 x1818247335203840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335203840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335203840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016635.382361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016635.382373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59cbb80 x1818247335203968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335203968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335203968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016635.382407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016635.382418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59c9c00 x1818247335204096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335204096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382446:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016635.382451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016635.382463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016635.382469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016635/real 1734016635] req@ffff8800b59c8000 x1818247335204224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016635.382479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335204224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016635.382486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016635.382489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016635.382492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016636.213936:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016636.249970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016637.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016638.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016638.253971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016639.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016640.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016640.257915:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016640.390002:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335204352:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016640.390071:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390095:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59ca680 x1818247335204352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390119:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335204352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390133:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390138:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390144:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335204352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016640.390171:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335204480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016640.390196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59ca300 x1818247335204480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335204480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335204480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016640.390254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335204608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016640.390276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59c9500 x1818247335204608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335204608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335204608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016640.390334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335204736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016640.390363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59c8e00 x1818247335204736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335204736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335204736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016640.390402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016640.390416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59cb100 x1818247335204864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335204864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016640.390463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016640.390476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59cb480 x1818247335204992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335204992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335204992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016640.390509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335205120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016640.390520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016640.390526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016640/real 1734016640] req@ffff8800b59c9f80 x1818247335205120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016640.390537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335205120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016640.390545:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016640.390546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016640.390549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335205120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016641.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016642.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016642.261949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016643.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016644.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016644.265896:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016645.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016645.398064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335205248:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016645.398131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398153:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59c8700 x1818247335205248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398173:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335205248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398183:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398187:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398192:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335205248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016645.398199:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335205376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016645.398215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398223:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59caa00 x1818247335205376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335205376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398242:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335205376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016645.398253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335205504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016645.398266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59c8a80 x1818247335205504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335205504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398291:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335205504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016645.398326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335205632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016645.398338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59c9f80 x1818247335205632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335205632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335205632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016645.398380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335205760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016645.398391:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398397:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59cb480 x1818247335205760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398407:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335205760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335205760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016645.398425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335205888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016645.398436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59cb100 x1818247335205888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335205888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335205888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016645.398469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016645.398481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016645.398487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016645/real 1734016645] req@ffff8800b59c8e00 x1818247335206016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016645.398496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335206016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016645.398503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016645.398505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016645.398508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016646.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016646.269952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016647.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016648.273924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016650.277944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016650.406043:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335206144:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016650.406108:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406129:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59c9500 x1818247335206144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406149:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335206144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406159:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406163:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406167:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335206144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016650.406178:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335206272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016650.406194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59ca300 x1818247335206272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335206272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335206272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016650.406232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335206400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016650.406244:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59ca680 x1818247335206400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406260:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335206400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406269:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335206400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016650.406298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335206528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016650.406310:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59c8000 x1818247335206528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335206528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335206528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016650.406345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016650.406357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59c9c00 x1818247335206656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335206656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016650.406390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016650.406402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59cbb80 x1818247335206784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406417:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335206784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406424:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406427:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016650.406476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016650.406493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016650.406500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016650/real 1734016650] req@ffff8800b59c9880 x1818247335206912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016650.406511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335206912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016650.406518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016650.406520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016650.406523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335206912:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016652.281953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016654.285955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016655.414054:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335207040:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016655.414118:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414140:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59cb800 x1818247335207040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335207040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414173:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414178:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335207040:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016655.414188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335207168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016655.414205:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59c9180 x1818247335207168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335207168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414234:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335207168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016655.414243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335207296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016655.414256:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59cad80 x1818247335207296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335207296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335207296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016655.414310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335207424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016655.414322:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59c9880 x1818247335207424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414340:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335207424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335207424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016655.414404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335207552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016655.414420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59cbb80 x1818247335207552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335207552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335207552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016655.414456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335207680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016655.414468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59c9c00 x1818247335207680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335207680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335207680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016655.414502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335207808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016655.414514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016655.414520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016655/real 1734016655] req@ffff8800b59c8000 x1818247335207808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016655.414530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335207808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016655.414537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016655.414539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016655.414542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335207808:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016656.289962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016658.293979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016660.297963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016660.422048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335207936:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016660.422114:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422136:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59ca680 x1818247335207936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422156:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335207936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422166:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422170:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422174:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335207936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016660.422183:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016660.422200:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59ca300 x1818247335208064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335208064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016660.422239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016660.422252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59c9500 x1818247335208192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335208192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016660.422351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016660.422367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422374:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59c8e00 x1818247335208320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335208320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016660.422403:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335208448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016660.422415:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59cb100 x1818247335208448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335208448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335208448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016660.422448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335208576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016660.422460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59cb480 x1818247335208576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335208576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335208576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016660.422494:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335208704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016660.422505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016660.422511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016660/real 1734016660] req@ffff8800b59c9f80 x1818247335208704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016660.422521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335208704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016660.422528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016660.422530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016660.422533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335208704:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016662.301960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016664.305940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016665.430053:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335208832:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016665.430120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430142:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59c8a80 x1818247335208832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335208832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430172:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430175:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335208832:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016665.430220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016665.430246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59caa00 x1818247335208960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430265:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335208960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335208960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016665.430285:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016665.430299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59c8700 x1818247335209088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335209088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016665.430353:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016665.430365:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59c9f80 x1818247335209216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335209216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430395:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016665.430400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335209344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016665.430412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59cb480 x1818247335209344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335209344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335209344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016665.430444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335209472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016665.430456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59cb100 x1818247335209472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335209472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335209472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016665.430490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335209600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016665.430502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016665.430507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016665/real 1734016665] req@ffff8800b59c8e00 x1818247335209600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016665.430517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335209600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016665.430524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016665.430526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016665.430529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335209600:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016666.309953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016668.313985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016670.317932:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016670.438067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335209728:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016670.438161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59c9500 x1818247335209728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438204:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335209728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335209728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016670.438229:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016670.438247:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59ca300 x1818247335209856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335209856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016670.438292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016670.438306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438313:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59ca680 x1818247335209984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335209984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438331:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335209984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016670.438361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335210112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016670.438373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59c8000 x1818247335210112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335210112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335210112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016670.438407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335210240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016670.438419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59c9c00 x1818247335210240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335210240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335210240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016670.438452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335210368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016670.438464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59cbb80 x1818247335210368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335210368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335210368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016670.438498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335210496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016670.438509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016670.438515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016670/real 1734016670] req@ffff8800b59c9880 x1818247335210496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016670.438525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335210496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016670.438532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016670.438534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016670.438537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335210496:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016672.321947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016675.446052:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335210624:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016675.446120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446141:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59cad80 x1818247335210624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335210624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335210624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016675.446186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335210752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016675.446203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59c9180 x1818247335210752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335210752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335210752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016675.446241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335210880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016675.446254:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59cb800 x1818247335210880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335210880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335210880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016675.446309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016675.446321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59c9880 x1818247335211008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335211008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016675.446362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335211136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016675.446374:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59cbb80 x1818247335211136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335211136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335211136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016675.446406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335211264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016675.446418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59c9c00 x1818247335211264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335211264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335211264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016675.446452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335211392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016675.446463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016675.446469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016675/real 1734016675] req@ffff8800b59c8000 x1818247335211392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016675.446479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335211392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016675.446486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016675.446488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016675.446491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335211392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016680.454109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335211520:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016680.454179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454203:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59ca680 x1818247335211520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454223:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335211520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454235:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454244:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335211520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016680.454252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016680.454274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59ca300 x1818247335211648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335211648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016680.454313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016680.454327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59c9500 x1818247335211776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335211776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016680.454384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016680.454396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59c8e00 x1818247335211904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335211904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335211904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016680.454439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016680.454451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59cb100 x1818247335212032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335212032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016680.454486:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016680.454497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59cb480 x1818247335212160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454513:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335212160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454520:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016680.454532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016680.454544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016680.454550:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016680/real 1734016680] req@ffff8800b59c9f80 x1818247335212288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016680.454559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335212288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016680.454566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016680.454568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016680.454571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016681.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016682.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016683.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016684.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016685.462096:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335212416:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016685.462179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59c8700 x1818247335212416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335212416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335212416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016685.462254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335212544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016685.462273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59caa00 x1818247335212544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335212544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462299:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335212544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016685.462311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335212672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016685.462325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59c8a80 x1818247335212672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335212672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335212672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016685.462384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335212800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016685.462397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59c9f80 x1818247335212800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335212800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335212800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016685.462433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016685.462445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59cb480 x1818247335212928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335212928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335212928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016685.462477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016685.462489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59cb100 x1818247335213056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335213056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016685.462532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016685.462544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016685.462550:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016685/real 1734016685] req@ffff8800b59c8e00 x1818247335213184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016685.462559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335213184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016685.462566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016685.462568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016685.462571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016690.470013:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335213312:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016690.470136:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335213440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016690.470177:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335213568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016690.470196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335213696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016690.470213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016690.470250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016690.470308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214080:192.168.201.104@tcp:8:kworker.0 00000400:00000100:1.0:1734016693.475947:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734016693.475993:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016693.476011:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016693.476046:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016693.476053:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016693.476060:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016693.476068:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016693.476076:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016693.476082:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:2.0:1734016693.476148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59c9880 x1818247335214080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335214080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476189:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476193:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016693.476207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59cbb80 x1818247335213952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335213952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476226:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016693.476238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59c9c00 x1818247335213824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476248:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335213824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335213824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016693.476271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59ca300 x1818247335213440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335213440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335213440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016693.476300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59ca680 x1818247335213568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335213568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335213568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016693.476329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59c8000 x1818247335213696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335213696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476346:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476349:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476352:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335213696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016693.476360:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016690/real 1734016693] req@ffff8800b59c9500 x1818247335213312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016693.476371:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335213312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016693.476379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016693.476381:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016693.476384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335213312:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016695.478111:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335214208:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016695.478183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59cb800 x1818247335214208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478225:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335214208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335214208:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016695.478255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335214336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016695.478273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59c9180 x1818247335214336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335214336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335214336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016695.478320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335214464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016695.478333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59cad80 x1818247335214464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335214464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335214464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016695.478389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335214592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016695.478401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478407:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59c9500 x1818247335214592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335214592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335214592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016695.478438:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016695.478451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59c8000 x1818247335214720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335214720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016695.478487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016695.478498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59ca680 x1818247335214848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335214848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016695.478532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016695.478544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016695.478550:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016695/real 1734016695] req@ffff8800b59ca300 x1818247335214976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016695.478561:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335214976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016695.478569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016695.478571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016695.478574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335214976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016696.213955:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016697.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016698.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016699.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016700.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016700.486014:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335215104:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016700.486081:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486101:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59c9c00 x1818247335215104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486121:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335215104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486131:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486134:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486139:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335215104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016700.486146:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335215232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016700.486163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486169:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59cbb80 x1818247335215232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486189:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335215232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486197:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486199:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486202:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335215232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016700.486208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335215360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016700.486221:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486227:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59c9880 x1818247335215360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335215360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335215360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016700.486276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335215488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016700.486288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486294:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59c8e00 x1818247335215488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335215488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335215488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016700.486324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335215616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016700.486336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486342:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59cb100 x1818247335215616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335215616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335215616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016700.486369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335215744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016700.486380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59cb480 x1818247335215744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335215744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335215744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016700.486412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335215872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016700.486427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016700.486433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016700/real 1734016700] req@ffff8800b59c9f80 x1818247335215872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016700.486443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335215872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016700.486450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016700.486452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016700.486455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335215872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016701.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016702.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016703.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016704.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016705.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016705.494058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335216000:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016705.494125:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59c8a80 x1818247335216000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494167:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335216000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494177:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494181:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335216000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016705.494192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335216128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016705.494213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59caa00 x1818247335216128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335216128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494244:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335216128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016705.494250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335216256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016705.494263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59c8700 x1818247335216256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335216256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335216256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016705.494317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335216384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016705.494330:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494336:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59c9f80 x1818247335216384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335216384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494364:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494367:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335216384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016705.494372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335216512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016705.494383:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59cb480 x1818247335216512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335216512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494406:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494408:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335216512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016705.494416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335216640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016705.494427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59cb100 x1818247335216640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335216640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335216640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016705.494462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335216768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016705.494474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016705.494479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016705/real 1734016705] req@ffff8800b59c8e00 x1818247335216768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016705.494489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335216768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016705.494549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016705.494552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016705.494556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335216768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016706.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016707.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016708.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016709.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016710.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016710.502063:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335216896:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016710.502129:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502150:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59c9880 x1818247335216896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502170:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335216896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502180:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502184:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502189:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335216896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016710.502197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016710.502212:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59cbb80 x1818247335217024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335217024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502244:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016710.502250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016710.502262:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59c9c00 x1818247335217152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335217152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016710.502316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016710.502331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59ca300 x1818247335217280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335217280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502355:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016710.502365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335217408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016710.502376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59ca680 x1818247335217408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335217408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502404:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335217408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016710.502409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335217536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016710.502467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59c8000 x1818247335217536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335217536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335217536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016710.502510:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335217664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016710.502522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016710.502528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016710/real 1734016710] req@ffff8800b59c9500 x1818247335217664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016710.502537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335217664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016710.502545:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016710.502547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016710.502550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335217664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016711.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016712.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016713.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016714.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016715.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016715.510027:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335217792:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016715.510093:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510115:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59cad80 x1818247335217792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510134:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335217792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510145:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510149:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510154:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335217792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016715.510161:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016715.510178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59c9180 x1818247335217920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335217920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335217920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016715.510214:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016715.510227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59cb800 x1818247335218048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335218048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016715.510282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016715.510294:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59c9500 x1818247335218176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335218176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016715.510375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335218304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016715.510391:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59c8000 x1818247335218304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335218304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335218304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016715.510439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335218432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016715.510455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59ca680 x1818247335218432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335218432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335218432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016715.510496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335218560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016715.510509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016715.510515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016715/real 1734016715] req@ffff8800b59ca300 x1818247335218560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016715.510524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335218560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016715.510532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016715.510534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016715.510537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335218560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016720.518052:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335218688:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016720.518119:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518141:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59c9c00 x1818247335218688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518168:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335218688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518178:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518181:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335218688:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016720.518192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016720.518208:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59cbb80 x1818247335218816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335218816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016720.518292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016720.518312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518320:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59c9880 x1818247335218944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335218944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335218944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016720.518376:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016720.518388:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59c8e00 x1818247335219072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335219072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016720.518422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335219200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016720.518434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59cb100 x1818247335219200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335219200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335219200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016720.518468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335219328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016720.518483:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59cb480 x1818247335219328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335219328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335219328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016720.518516:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335219456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016720.518527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016720.518533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016720/real 1734016720] req@ffff8800b59c9f80 x1818247335219456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016720.518542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335219456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016720.518549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016720.518551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016720.518554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335219456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016725.526070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335219584:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016725.526138:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526187:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59c8700 x1818247335219584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526215:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335219584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335219584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016725.526241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016725.526261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59caa00 x1818247335219712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335219712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016725.526299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016725.526312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59c8a80 x1818247335219840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335219840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016725.526367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016725.526379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59c9f80 x1818247335219968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335219968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335219968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016725.526413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016725.526425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59cb480 x1818247335220096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335220096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016725.526458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016725.526472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59cb100 x1818247335220224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335220224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016725.526505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016725.526517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016725.526523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016725/real 1734016725] req@ffff8800b59c8e00 x1818247335220352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016725.526532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335220352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016725.526539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016725.526541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016725.526544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016730.534047:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335220480:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016730.534146:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534168:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59c9880 x1818247335220480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534194:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335220480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534205:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534208:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534213:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335220480:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016730.534221:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335220608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016730.534238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59cbb80 x1818247335220608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534256:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335220608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335220608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016730.534274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335220736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016730.534287:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59c9c00 x1818247335220736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335220736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335220736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016730.534340:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335220864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016730.534352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59ca300 x1818247335220864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335220864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534381:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335220864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016730.534386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016730.534397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59ca680 x1818247335220992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335220992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534420:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335220992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016730.534430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335221120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016730.534444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59c8000 x1818247335221120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335221120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335221120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016730.534477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335221248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016730.534488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016730.534494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016730/real 1734016730] req@ffff8800b59c9500 x1818247335221248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016730.534504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335221248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016730.534510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016730.534512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016730.534515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335221248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016735.542076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335221376:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016735.542143:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542164:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59cb800 x1818247335221376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335221376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542201:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335221376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016735.542216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335221504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016735.542233:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59c9180 x1818247335221504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335221504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335221504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016735.542286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335221632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016735.542299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59cad80 x1818247335221632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335221632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335221632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016735.542335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335221760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016735.542346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59c9500 x1818247335221760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335221760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335221760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016735.542380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335221888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016735.542391:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542397:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59c8000 x1818247335221888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335221888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335221888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016735.542425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016735.542440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59ca680 x1818247335222016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335222016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016735.542473:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016735.542484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016735.542490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016735/real 1734016735] req@ffff8800b59ca300 x1818247335222144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016735.542500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335222144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016735.542507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016735.542509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016735.542512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016737.213912:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016738.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016739.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016740.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016740.550120:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335222272:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016740.550192:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59c9c00 x1818247335222272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335222272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335222272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016740.550265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335222400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016740.550283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550290:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59cbb80 x1818247335222400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335222400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335222400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016740.550322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335222528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016740.550336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550342:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59c9880 x1818247335222528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550352:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9880 x1818247335222528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550366:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335222528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016740.550400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335222656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016740.550413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59c8e00 x1818247335222656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335222656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335222656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016740.550449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016740.550461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59cb100 x1818247335222784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335222784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016740.550496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016740.550508:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59cb480 x1818247335222912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335222912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335222912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016740.550542:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016740.550554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016740.550560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016740/real 1734016740] req@ffff8800b59c9f80 x1818247335223040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016740.550570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335223040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016740.550577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016740.550579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016740.550583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016741.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016741.459965:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016742.213772:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016743.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016743.463953:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016744.213764:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016745.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016745.467959:0:4737:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016745.558074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335223168:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016745.558141:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558162:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59c8a80 x1818247335223168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558181:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335223168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558191:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335223168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016745.558206:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335223296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016745.558223:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59caa00 x1818247335223296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558241:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335223296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335223296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016745.558260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335223424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016745.558277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59c8700 x1818247335223424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558294:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335223424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335223424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016745.558332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335223552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016745.558344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59c9f80 x1818247335223552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335223552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335223552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016745.558377:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016745.558389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59cb480 x1818247335223680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335223680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016745.558428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016745.558440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59cb100 x1818247335223808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335223808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558462:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558464:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558467:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016745.558472:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016745.558484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016745.558490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016745/real 1734016745] req@ffff8800b59c8e00 x1818247335223936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016745.558499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335223936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016745.558506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016745.558509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016745.558512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335223936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016746.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016747.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016747.471991:0:4737:0:(ldlm_lockd.c:2838:ldlm_bl_get_work()) Got 2 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1734016747.472006:0:4737:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3180/0xd17286ca2ee49827 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400020000 nid: local remote: 0x979ba2bc5598ad83 expref: -99 pid: 9551 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1734016748.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016749.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734016749.477052:0:1971:0:(client.c:1722:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8800a98b1c00 x1818247335224192/t0(0) o103->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016750.477933:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016750.566016:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335224320:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016750.566088:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566111:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59cbb80 x1818247335224320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566134:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335224320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566148:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566153:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566159:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335224320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016750.566187:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335224448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016750.566212:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59c9c00 x1818247335224448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335224448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335224448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016750.566273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335224576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016750.566296:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59ca300 x1818247335224576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566324:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335224576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566340:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335224576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016750.566355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335224704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016750.566377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59ca680 x1818247335224704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335224704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335224704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016750.566422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335224832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016750.566434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59c8000 x1818247335224832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335224832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335224832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016750.566482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335224960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016750.566494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59c9500 x1818247335224960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335224960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335224960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016750.566529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016750.566540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016750.566546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016750/real 1734016750] req@ffff8800b59cad80 x1818247335225088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016750.566556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335225088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016750.566563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016750.566565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016750.566568:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016751.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016751.479926:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016752.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016753.213905:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016753.483949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016754.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016754.485950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016755.213932:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016755.574107:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335225216:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016755.574208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335225344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016755.574231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335225472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016755.574255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335225600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016755.574272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016755.574294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016755.574310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016756.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016756.489948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016757.491969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016758.580157:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59c9500 x1818247335225600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335225600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580200:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580210:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335225600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016758.580217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59c8000 x1818247335225728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335225728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580244:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016758.580250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59ca680 x1818247335225856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335225856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580269:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016758.580280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59ca300 x1818247335225984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335225984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335225984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016758.580309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59c9180 x1818247335225216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335225216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335225216:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016758.580346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59cb800 x1818247335225344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335225344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335225344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016758.580386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016755/real 1734016758] req@ffff8800b59cad80 x1818247335225472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016758.580402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335225472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016758.580427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016758.580431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016758.580436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335225472:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016759.495987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016760.497980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016760.582074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335226112:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016760.582150:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59c9c00 x1818247335226112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582201:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335226112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582213:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335226112:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016760.582230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335226240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016760.582249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59cbb80 x1818247335226240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335226240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335226240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016760.582288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335226368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016760.582301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59c8e00 x1818247335226368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335226368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335226368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016760.582359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335226496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016760.582372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59cb100 x1818247335226496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335226496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335226496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016760.582407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335226624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016760.582419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59cb480 x1818247335226624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335226624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335226624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016760.582453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335226752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016760.582470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59c9f80 x1818247335226752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335226752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335226752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016760.582505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335226880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016760.582517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016760.582523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016760/real 1734016760] req@ffff8800b59c8700 x1818247335226880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016760.582533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335226880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016760.582540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016760.582542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016760.582545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335226880:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016762.501948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016763.503963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016765.507999:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016765.590095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335227008:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016765.590167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590191:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59caa00 x1818247335227008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335227008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335227008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016765.590248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335227136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016765.590266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59c8a80 x1818247335227136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335227136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590298:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335227136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016765.590304:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335227264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016765.590318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59c8700 x1818247335227264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335227264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335227264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016765.590376:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335227392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016765.590389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59c9f80 x1818247335227392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335227392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335227392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016765.590425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335227520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016765.590437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59cb480 x1818247335227520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335227520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335227520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016765.590471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335227648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016765.590538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59cb100 x1818247335227648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335227648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335227648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016765.590577:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335227776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016765.590590:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016765.590596:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016765/real 1734016765] req@ffff8800b59c8e00 x1818247335227776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016765.590606:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335227776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016765.590613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016765.590616:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016765.590619:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335227776:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016766.509954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016768.513934:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016769.515969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016770.598036:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335227904:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016770.598103:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598124:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59cbb80 x1818247335227904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598152:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335227904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598162:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598165:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598170:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335227904:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016770.598177:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016770.598194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59c9c00 x1818247335228032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335228032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598219:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598222:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598225:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016770.598231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016770.598244:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59cad80 x1818247335228160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598260:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335228160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016770.598299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016770.598312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59cb800 x1818247335228288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335228288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016770.598345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335228416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016770.598357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59c9180 x1818247335228416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335228416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598381:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335228416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016770.598442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335228544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016770.598462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59ca300 x1818247335228544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335228544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335228544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016770.598499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335228672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016770.598511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016770.598516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016770/real 1734016770] req@ffff8800b59ca680 x1818247335228672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016770.598526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335228672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016770.598534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016770.598536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016770.598539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335228672:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016771.519980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016772.521991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016774.525920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016775.527959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016775.606016:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335228800:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016775.606083:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606104:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59c8000 x1818247335228800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606132:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335228800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606142:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606145:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606150:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335228800:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016775.606157:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016775.606173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606179:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59c9500 x1818247335228928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335228928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606198:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606201:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606204:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335228928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016775.606210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016775.606223:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606229:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59ca680 x1818247335229056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335229056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606249:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016775.606278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016775.606291:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59ca300 x1818247335229184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335229184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016775.606361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335229312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016775.606377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59c9180 x1818247335229312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335229312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335229312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016775.606413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335229440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016775.606427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59cb800 x1818247335229440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335229440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335229440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016775.606461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335229568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016775.606472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016775.606479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016775/real 1734016775] req@ffff8800b59cad80 x1818247335229568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016775.606489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335229568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016775.606495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016775.606497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016775.606500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335229568:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016777.531885:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016778.533977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016780.537990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016780.614055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335229696:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016780.614138:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614163:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59c9c00 x1818247335229696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614192:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335229696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614211:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335229696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016780.614259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016780.614281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614290:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59cbb80 x1818247335229824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335229824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016780.614322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016780.614337:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59c8e00 x1818247335229952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335229952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335229952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016780.614398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016780.614411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59cb100 x1818247335230080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335230080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614435:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614440:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016780.614445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335230208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016780.614456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59cb480 x1818247335230208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335230208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335230208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016780.614489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335230336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016780.614504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59c9f80 x1818247335230336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335230336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335230336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016780.614538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335230464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016780.614550:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016780.614555:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016780/real 1734016780] req@ffff8800b59c8700 x1818247335230464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016780.614565:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335230464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016780.614572:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016780.614574:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016780.614577:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335230464:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016781.539952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016784.545968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016785.622091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335230592:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016785.622208:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59c8a80 x1818247335230592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335230592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335230592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016785.622289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016785.622308:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59caa00 x1818247335230720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335230720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622333:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622336:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016785.622367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016785.622381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59c8700 x1818247335230848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335230848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622408:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622412:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016785.622418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016785.622429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59c9f80 x1818247335230976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335230976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335230976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016785.622462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335231104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016785.622474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59cb480 x1818247335231104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622490:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335231104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335231104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016785.622509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335231232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016785.622524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59cb100 x1818247335231232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335231232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335231232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016785.622558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335231360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016785.622570:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016785.622576:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016785/real 1734016785] req@ffff8800b59c8e00 x1818247335231360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016785.622587:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335231360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016785.622594:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016785.622596:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016785.622599:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335231360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016786.213923:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016787.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016787.551927:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016788.213778:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016789.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016790.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016790.557970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016790.630103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335231488:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016790.630197:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59cbb80 x1818247335231488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335231488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630268:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630272:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335231488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016790.630284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335231616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016790.630304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59c9c00 x1818247335231616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335231616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335231616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016790.630342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335231744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016790.630357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59cad80 x1818247335231744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335231744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335231744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016790.630419:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335231872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016790.630432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59cb800 x1818247335231872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335231872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335231872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016790.630468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016790.630481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59c9180 x1818247335232000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335232000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016790.630517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016790.630533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59ca300 x1818247335232128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630550:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335232128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630560:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630563:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016790.630568:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016790.630581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016790.630587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016790/real 1734016790] req@ffff8800b59ca680 x1818247335232256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016790.630596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335232256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016790.630603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016790.630605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016790.630608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016791.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016792.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016793.563964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016795.638076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335232384:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016795.638171:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59c9500 x1818247335232384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335232384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638246:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335232384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016795.638260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335232512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016795.638280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638286:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59c8000 x1818247335232512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335232512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335232512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016795.638345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335232640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016795.638360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638366:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59ca680 x1818247335232640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638376:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335232640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335232640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016795.638398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335232768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016795.638411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59ca300 x1818247335232768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335232768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335232768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016795.638445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016795.638459:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59c9180 x1818247335232896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335232896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335232896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016795.638494:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016795.638511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59cb800 x1818247335233024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335233024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016795.638546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016795.638559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016795.638565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016795/real 1734016795] req@ffff8800b59cad80 x1818247335233152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016795.638575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335233152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016795.638582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016795.638584:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016795.638588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233152:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016796.569898:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016799.575972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016800.646095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335233280:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016800.646190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59c9c00 x1818247335233280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335233280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646266:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335233280:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016800.646278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335233408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016800.646297:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59cbb80 x1818247335233408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335233408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335233408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016800.646335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335233536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016800.646350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59c8e00 x1818247335233536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335233536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335233536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016800.646414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335233664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016800.646428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59cb100 x1818247335233664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335233664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335233664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016800.646464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016800.646476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59cb480 x1818247335233792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335233792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016800.646511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016800.646527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59c9f80 x1818247335233920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335233920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335233920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016800.646563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016800.646575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016800.646581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016800/real 1734016800] req@ffff8800b59c8700 x1818247335234048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016800.646591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335234048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016800.646598:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016800.646601:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016800.646604:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234048:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016802.581984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016805.588000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016805.654068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335234176:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016805.654163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59caa00 x1818247335234176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335234176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335234176:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016805.654250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335234304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016805.654270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654276:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59c8a80 x1818247335234304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654288:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335234304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335234304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016805.654333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335234432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016805.654348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59c8700 x1818247335234432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335234432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335234432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016805.654386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335234560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016805.654399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59c9f80 x1818247335234560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335234560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335234560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016805.654434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016805.654448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59cb480 x1818247335234688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335234688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016805.654484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016805.654500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59cb100 x1818247335234816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335234816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016805.654534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016805.654546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016805.654552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016805/real 1734016805] req@ffff8800b59c8e00 x1818247335234944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016805.654562:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335234944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016805.654569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016805.654571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016805.654574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335234944:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016808.593962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016810.662044:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335235072:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016810.662113:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59cbb80 x1818247335235072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662162:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335235072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662172:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662176:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662180:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335235072:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016810.662188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335235200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016810.662204:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59c9c00 x1818247335235200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335235200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662236:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335235200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016810.662260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335235328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016810.662274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59cad80 x1818247335235328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335235328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335235328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016810.662311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335235456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016810.662322:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662328:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59cb800 x1818247335235456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335235456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662346:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335235456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016810.662357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335235584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016810.662369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59c9180 x1818247335235584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335235584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335235584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016810.662404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335235712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016810.662418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59ca300 x1818247335235712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335235712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335235712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016810.662452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335235840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016810.662464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016810.662470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016810/real 1734016810] req@ffff8800b59ca680 x1818247335235840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016810.662479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335235840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016810.662486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016810.662488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016810.662491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335235840:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016811.599953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016814.605954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016815.670100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335235968:192.168.201.104@tcp:250:kworker.0 00000800:00000100:1.0:1734016815.670176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59c8000 x1818247335235968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335235968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335235968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016815.670256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016815.670273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59c9500 x1818247335236096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335236096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016815.670313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016815.670326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59ca680 x1818247335236224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335236224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016815.670385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016815.670398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59ca300 x1818247335236352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335236352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016815.670434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335236480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734016815.670446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59c9180 x1818247335236480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335236480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335236480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016815.670481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335236608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016815.670497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59cb800 x1818247335236608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335236608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335236608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016815.670532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335236736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734016815.670544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734016815.670550:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016815/real 1734016815] req@ffff8800b59cad80 x1818247335236736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734016815.670559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335236736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734016815.670567:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734016815.670569:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734016815.670572:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335236736:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016817.611973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016819.213930:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016820.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016820.617956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1734016820.678066:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335236864:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734016820.678162:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335236992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016820.678183:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335237120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016820.678208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335237248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016820.678223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335237376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734016820.678237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335237504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734016820.678251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335237632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016821.213786:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016822.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016823.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016823.623914:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1734016823.683973:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734016823.684021:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734016823.684041:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016823.684079:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016823.684091:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016823.684102:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016823.684113:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016823.684124:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734016823.684134:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:2.0:1734016824.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016825.686028:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335237760:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016825.686095:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686117:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59c8e00 x1818247335237760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686136:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335237760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686147:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686150:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686155:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335237760:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016825.686162:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335237888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016825.686178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686185:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59cbb80 x1818247335237888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686196:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335237888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335237888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016825.686215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016825.686229:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59c9c00 x1818247335238016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335238016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016825.686288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016825.686300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59c8700 x1818247335238144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335238144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016825.686334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335238272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016825.686346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59c9f80 x1818247335238272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335238272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335238272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016825.686380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335238400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016825.686392:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59cb480 x1818247335238400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335238400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335238400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016825.686425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335238528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016825.686436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016825.686442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016825/real 1734016825] req@ffff8800b59cb100 x1818247335238528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016825.686512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335238528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016825.686521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016825.686523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016825.686526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335238528:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016826.629938:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016829.635913:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016830.694041:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335238656:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016830.694109:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694131:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59c8a80 x1818247335238656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694151:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335238656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694161:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694165:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694169:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335238656:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016830.694177:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016830.694193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59caa00 x1818247335238784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335238784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694218:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694224:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016830.694230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016830.694243:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694249:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59cb100 x1818247335238912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335238912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694279:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335238912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016830.694306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016830.694318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59cb480 x1818247335239040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335239040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694344:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016830.694352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335239168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016830.694405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59c9f80 x1818247335239168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335239168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694432:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694434:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335239168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016830.694449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335239296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016830.694464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59c8700 x1818247335239296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335239296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335239296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016830.694498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335239424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016830.694510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016830.694516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016830/real 1734016830] req@ffff8800b59c9c00 x1818247335239424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016830.694525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335239424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016830.694532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016830.694534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016830.694537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335239424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016831.213914:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016832.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016832.641914:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016833.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016834.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016835.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016835.647842:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016835.702007:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335239552:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016835.702078:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702102:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59cbb80 x1818247335239552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702126:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335239552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702140:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702145:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702151:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335239552:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016835.702178:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016835.702204:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59c8e00 x1818247335239680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335239680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016835.702265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016835.702327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59cad80 x1818247335239808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335239808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016835.702394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016835.702415:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59cb800 x1818247335239936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335239936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335239936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016835.702453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016835.702465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59c9180 x1818247335240064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335240064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016835.702521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016835.702533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59ca300 x1818247335240192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335240192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016835.702567:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016835.702579:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016835.702585:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016835/real 1734016835] req@ffff8800b59ca680 x1818247335240320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016835.702594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335240320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016835.702601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016835.702603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016835.702606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016836.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016838.653854:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016840.213941:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016840.710072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335240448:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016840.710170:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59c9500 x1818247335240448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335240448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335240448:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016840.710281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335240576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016840.710302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59c8000 x1818247335240576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335240576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335240576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016840.710341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335240704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016840.710360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59ca680 x1818247335240704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335240704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335240704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016840.710425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335240832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016840.710437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59ca300 x1818247335240832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335240832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710462:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335240832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016840.710474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016840.710487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59c9180 x1818247335240960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335240960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335240960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016840.710530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016840.710542:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59cb800 x1818247335241088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710558:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335241088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016840.710576:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016840.710589:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016840.710596:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016840/real 1734016840] req@ffff8800b59cad80 x1818247335241216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016840.710606:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335241216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016840.710613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016840.710617:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016840.710620:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241216:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016841.659952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016844.665934:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016845.718045:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335241344:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016845.718148:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718171:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59c8e00 x1818247335241344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335241344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718201:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335241344:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016845.718217:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335241472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016845.718237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59cbb80 x1818247335241472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335241472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718266:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335241472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016845.718278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335241600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016845.718291:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59c9c00 x1818247335241600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335241600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718317:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335241600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016845.718345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335241728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016845.718358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59c8700 x1818247335241728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335241728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335241728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016845.718392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016845.718403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59c9f80 x1818247335241856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335241856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016845.718437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016845.718448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59cb480 x1818247335241984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335241984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335241984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016845.718481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335242112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016845.718493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016845.718499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016845/real 1734016845] req@ffff8800b59cb100 x1818247335242112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016845.718508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335242112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016845.718515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016845.718518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016845.718521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335242112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016846.213897:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016847.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016847.671941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016848.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016850.677982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016850.726113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335242240:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016850.726199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59caa00 x1818247335242240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726248:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335242240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335242240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016850.726275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335242368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016850.726293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59c8a80 x1818247335242368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335242368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726327:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335242368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016850.726333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335242496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016850.726348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59cb100 x1818247335242496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335242496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335242496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016850.726408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335242624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016850.726421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59cb480 x1818247335242624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335242624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335242624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016850.726456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335242752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016850.726468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59c9f80 x1818247335242752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335242752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335242752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016850.726503:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335242880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016850.726515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59c8700 x1818247335242880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335242880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335242880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016850.726548:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016850.726561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016850.726567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016850/real 1734016850] req@ffff8800b59c9c00 x1818247335243008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016850.726577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335243008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016850.726592:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016850.726595:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016850.726598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243008:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016853.683952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016855.734032:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335243136:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016855.734099:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734120:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59cbb80 x1818247335243136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734140:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335243136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734151:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734154:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734159:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335243136:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016855.734166:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335243264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016855.734183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59c8e00 x1818247335243264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734201:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335243264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734215:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335243264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016855.734240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335243392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016855.734254:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59cad80 x1818247335243392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335243392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335243392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016855.734289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335243520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016855.734300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59cb800 x1818247335243520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734316:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335243520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335243520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016855.734334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016855.734346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734353:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59c9180 x1818247335243648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335243648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016855.734381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016855.734393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59ca300 x1818247335243776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335243776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016855.734426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016855.734438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016855.734444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016855/real 1734016855] req@ffff8800b59ca680 x1818247335243904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016855.734454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335243904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016855.734469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016855.734471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016855.734474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335243904:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016856.689831:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016859.695948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016860.742099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335244032:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016860.742166:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742188:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59c8000 x1818247335244032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742207:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335244032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742218:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335244032:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016860.742233:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335244160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016860.742249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59c9500 x1818247335244160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335244160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335244160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016860.742288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335244288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016860.742301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59ca680 x1818247335244288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335244288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742327:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742330:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335244288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016860.742356:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335244416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016860.742368:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742374:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59ca300 x1818247335244416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742384:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335244416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742397:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335244416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016860.742402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335244544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016860.742413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742420:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59c9180 x1818247335244544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335244544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335244544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016860.742447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335244672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016860.742458:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59cb800 x1818247335244672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335244672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335244672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016860.742491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335244800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016860.742502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016860.742508:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016860/real 1734016860] req@ffff8800b59cad80 x1818247335244800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016860.742518:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335244800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016860.742532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016860.742534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016860.742537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335244800:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016862.701974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016863.213918:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016864.213889:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016865.213890:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016865.707963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016865.750048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335244928:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016865.750121:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750146:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59c8e00 x1818247335244928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750169:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335244928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750184:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750189:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750195:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335244928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016865.750223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016865.750248:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59cbb80 x1818247335245056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335245056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016865.750319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016865.750343:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59c9c00 x1818247335245184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335245184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016865.750404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016865.750427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59c8700 x1818247335245312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335245312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750454:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016865.750466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335245440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016865.750479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59c9f80 x1818247335245440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335245440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335245440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016865.750528:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335245568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016865.750540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59cb480 x1818247335245568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335245568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335245568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016865.750574:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335245696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016865.750586:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016865.750592:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016865/real 1734016865] req@ffff8800b59cb100 x1818247335245696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016865.750602:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335245696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016865.750610:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016865.750612:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016865.750615:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335245696:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016868.713983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016870.758069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335245824:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016870.758135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758156:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59c8a80 x1818247335245824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758175:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335245824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758186:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758189:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758194:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335245824:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016870.758201:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016870.758217:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59caa00 x1818247335245952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335245952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335245952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016870.758262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016870.758275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59cb100 x1818247335246080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335246080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758304:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016870.758329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016870.758341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758347:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59cb480 x1818247335246208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335246208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758365:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016870.758376:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335246336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016870.758388:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758394:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59c9f80 x1818247335246336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335246336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335246336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016870.758422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335246464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016870.758434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59c8700 x1818247335246464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335246464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758461:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335246464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016870.758466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335246592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016870.758478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016870.758484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016870/real 1734016870] req@ffff8800b59c9c00 x1818247335246592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016870.758495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335246592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016870.758502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016870.758504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016870.758507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335246592:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016871.719954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016874.725924:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016875.766046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335246720:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016875.766112:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766133:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59cbb80 x1818247335246720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766152:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335246720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766163:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335246720:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016875.766179:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016875.766195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766202:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59c8e00 x1818247335246848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335246848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766222:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766225:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016875.766234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016875.766247:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766253:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59cad80 x1818247335246976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335246976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335246976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016875.766304:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335247104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016875.766317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766323:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59cb800 x1818247335247104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335247104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335247104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016875.766351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335247232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016875.766364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59c9180 x1818247335247232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335247232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335247232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016875.766397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335247360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016875.766408:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59ca300 x1818247335247360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766424:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335247360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766431:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335247360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016875.766441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335247488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016875.766453:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016875.766459:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016875/real 1734016875] req@ffff8800b59ca680 x1818247335247488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016875.766469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335247488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016875.766477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016875.766479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016875.766482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335247488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016876.213931:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016877.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016877.731942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016880.737964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016880.774098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335247616:192.168.201.104@tcp:250:kworker.0 00000800:00000100:2.0:1734016880.774183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59c9500 x1818247335247616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335247616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335247616:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016880.774257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335247744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016880.774274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59c8000 x1818247335247744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335247744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335247744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016880.774345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335247872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016880.774359:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59ca680 x1818247335247872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335247872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335247872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016880.774394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335248000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016880.774406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59ca300 x1818247335248000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335248000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335248000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016880.774441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335248128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734016880.774454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59c9180 x1818247335248128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335248128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335248128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016880.774486:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335248256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016880.774498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59cb800 x1818247335248256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335248256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335248256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016880.774576:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335248384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734016880.774596:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734016880.774603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734016880/real 1734016880] req@ffff8800b59cad80 x1818247335248384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734016880.774614:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335248384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734016905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734016880.774622:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734016880.774625:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734016880.774628:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335248384:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016883.743951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016885.782018:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335248512:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734016885.782122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335248640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016885.782152:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335248768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016885.782178:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335248896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016885.782200:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335249024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734016885.782263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335249152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734016885.782326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335249280:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734016886.749944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016889.755964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016892.761928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016895.767947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016898.773980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016901.779944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016904.785954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016907.792006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016910.797932:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016913.804003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016916.809823:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016919.815926:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016922.821950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016925.827968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016928.833945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016931.839941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016934.213917:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016934.845923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016935.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016936.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016937.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016937.851883:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016940.857956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016943.864033:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016946.869954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016949.875961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016952.881981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016953.213922:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734016953.891980:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734016953.892031:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734016953.892051:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016953.892088:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016953.892097:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016953.892105:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016953.892112:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016953.892119:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734016953.892128:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:2.0:1734016954.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016955.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016955.887991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1734016958.893977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016960.902047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335262336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016960.902062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016960.902066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016960.902067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016960.902071:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1225 00000100:00080000:0.0:1734016960.902073:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016960.902077:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016960.902081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016960.902089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335262464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016960.902096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016960.902099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016960.902101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016960.902104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1225 00000100:00080000:0.0:1734016960.902106:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016960.902110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016960.902114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016960.902121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335262592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016960.902129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016960.902132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016960.902133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016960.902136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1225 00000100:00080000:0.0:1734016960.902140:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016960.902144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016960.902180:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016960.902191:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335262720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734016961.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016961.899958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016962.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016963.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016964.213918:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016964.905953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016965.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016965.909838:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.909847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016965.909851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.909862:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.909871:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.909879:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016965.909914:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335262848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016965.909948:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.909953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016965.909954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.909959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.909962:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.909965:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016965.909975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016965.909985:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335262976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016965.910002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.910005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016965.910006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.910010:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.910013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.910016:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016965.910020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016965.910029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335263104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016965.910038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.910041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016965.910042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.910045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.910048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.910051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016965.910055:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016965.910096:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335263232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016965.910108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.910111:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016965.910113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.910117:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.910120:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.910124:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016965.910129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016965.910138:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335263360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016965.910147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.910150:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016965.910151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.910154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.910157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.910160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016965.910164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016965.910172:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335263488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016965.910179:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016965.910182:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016965.910184:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016965.910187:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1230 00000100:00080000:0.0:1734016965.910190:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016965.910193:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016965.910197:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016965.910205:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335263616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734016966.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016967.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016967.911963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016968.213936:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016969.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016970.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016970.917835:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.917847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016970.917852:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.917867:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.917879:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.917889:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016970.917937:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335263744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016970.917975:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.918005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016970.918007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.918012:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.918016:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.918020:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016970.918033:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016970.918046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335263872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016970.918057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.918060:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016970.918062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.918066:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.918070:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.918074:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016970.918079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016970.918090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335264000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016970.918100:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.918103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016970.918105:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.918109:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.918111:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.918115:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016970.918119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016970.918127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335264128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016970.918135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.918138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016970.918140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.918143:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.918146:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.918150:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016970.918154:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016970.918161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335264256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016970.918176:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.918180:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016970.918181:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.918186:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.918188:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.918192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016970.918196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016970.918204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335264384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016970.918212:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016970.918215:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016970.918216:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016970.918220:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1235 00000100:00080000:0.0:1734016970.918222:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016970.918226:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016970.918230:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016970.918238:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335264512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016970.918411:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016971.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016972.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016973.213906:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016973.924003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016974.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016975.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016975.925947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.925959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016975.925966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.925980:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.925993:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926002:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016975.926054:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335264640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016975.926095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.926100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016975.926102:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.926107:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.926110:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926114:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016975.926126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016975.926138:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335264768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016975.926148:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.926151:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016975.926153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.926157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.926160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016975.926169:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016975.926178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335264896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016975.926186:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.926190:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016975.926192:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.926196:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.926198:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926202:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016975.926206:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016975.926214:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335265024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016975.926223:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.926226:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016975.926228:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.926232:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.926234:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926238:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016975.926242:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016975.926250:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335265152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016975.926258:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.926261:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016975.926262:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.926266:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.926268:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926272:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016975.926277:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016975.926289:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335265280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016975.926297:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016975.926300:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016975.926302:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016975.926305:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1240 00000100:00080000:0.0:1734016975.926308:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016975.926311:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016975.926315:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016975.926323:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335265408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734016976.213975:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016976.929925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016977.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016978.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016979.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016979.935992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016980.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016980.933867:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.933875:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016980.933880:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.933890:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.933898:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.933906:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016980.933946:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335265536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016980.934001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.934008:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016980.934011:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.934018:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.934023:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.934029:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016980.934042:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016980.934062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335265664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016980.934078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.934084:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016980.934086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.934092:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.934096:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.934100:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016980.934105:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016980.934117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335265792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016980.934128:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.934131:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016980.934133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.934137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.934139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.934144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016980.934148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016980.934157:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335265920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016980.934165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.934168:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016980.934170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.934173:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.934176:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.934179:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016980.934183:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016980.934190:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335266048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016980.934198:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.934201:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016980.934202:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.934205:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.934208:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.934211:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016980.934215:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016980.934223:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335266176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016980.934231:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016980.934234:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016980.934236:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016980.934240:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1245 00000100:00080000:0.0:1734016980.934243:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016980.934246:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016980.934250:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016980.934257:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335266304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734016981.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734016982.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016982.941976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734016983.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016984.213951:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016985.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016985.941837:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.941847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016985.941851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.941862:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.941872:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.941879:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016985.941915:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335266432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016985.941945:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.941949:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016985.941950:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.941955:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.941958:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.941961:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016985.941972:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016985.941982:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335266560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016985.941992:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.941995:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016985.941997:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.942000:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.942003:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.942006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016985.942010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016985.942018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335266688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016985.942026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.942029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016985.942031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.942034:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.942036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.942040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016985.942044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016985.942051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335266816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016985.942061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.942064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016985.942066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.942069:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.942072:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.942075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016985.942079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016985.942087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335266944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016985.942095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.942097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016985.942099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.942102:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.942104:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.942107:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016985.942111:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016985.942118:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335267072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016985.942126:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016985.942129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016985.942130:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016985.942133:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1250 00000100:00080000:0.0:1734016985.942136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016985.942138:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016985.942142:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016985.942152:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335267200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734016985.947952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016986.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016987.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016988.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016988.953950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016989.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016990.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016990.949836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.949850:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016990.949855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.949867:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.949879:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.949889:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016990.949933:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335267328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016990.949982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.949987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016990.949989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.949994:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.949998:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.950003:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016990.950013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016990.950027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335267456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016990.950037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.950040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016990.950042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.950046:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.950048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.950051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016990.950057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016990.950066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335267584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016990.950076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.950079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016990.950081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.950084:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.950086:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.950090:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016990.950094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016990.950102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335267712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016990.950110:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.950113:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016990.950114:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.950118:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.950120:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.950124:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016990.950128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016990.950135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335267840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016990.950143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.950146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016990.950148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.950151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.950154:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.950158:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016990.950161:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016990.950168:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335267968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016990.950177:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016990.950179:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016990.950181:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016990.950184:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1255 00000100:00080000:0.0:1734016990.950187:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016990.950190:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016990.950193:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016990.950201:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335268096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734016991.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016991.959960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016992.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016993.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016994.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016994.965924:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016995.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734016995.957806:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.957816:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734016995.957820:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.957830:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.957839:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.957846:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734016995.957882:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335268224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016995.957924:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.957928:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016995.957930:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.957934:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.957937:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.957941:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016995.957950:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016995.957961:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335268352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016995.957970:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.957973:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016995.957975:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.957985:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.957988:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.957992:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016995.957996:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734016995.958004:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335268480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016995.958012:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.958015:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016995.958017:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.958020:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.958023:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.958027:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016995.958031:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016995.958039:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335268608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016995.958047:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.958051:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734016995.958052:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.958055:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.958058:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.958061:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016995.958065:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734016995.958072:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335268736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016995.958079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.958083:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734016995.958084:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.958088:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.958090:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.958093:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016995.958097:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016995.958106:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335268864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734016995.958114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734016995.958117:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734016995.958119:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734016995.958122:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1260 00000100:00080000:0.0:1734016995.958125:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734016995.958128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734016995.958132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734016995.958139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335268992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734016996.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016997.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734016997.971977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734016998.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734016999.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017000.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017000.965816:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.965825:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017000.965829:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.965840:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.965849:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.965857:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017000.965891:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335269120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017000.965926:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.965930:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017000.965932:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.965936:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.965939:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.965943:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017000.965952:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017000.965962:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335269248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017000.965973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.965976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017000.965978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.965981:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.965983:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.965986:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017000.965991:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017000.966000:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335269376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017000.966008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.966011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017000.966012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.966016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.966018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.966022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017000.966026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017000.966034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335269504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017000.966043:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.966046:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017000.966047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.966051:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.966054:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.966057:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017000.966061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017000.966069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335269632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017000.966076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.966079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017000.966081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.966084:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.966087:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.966090:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017000.966094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017000.966102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335269760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017000.966110:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017000.966112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017000.966114:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017000.966117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1265 00000100:00080000:0.0:1734017000.966119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017000.966123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017000.966127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017000.966134:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335269888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017000.977962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017001.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017002.213789:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017003.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017003.983986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017004.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017005.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017005.973836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.973844:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017005.973849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.973859:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.973869:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.973876:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017005.973913:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335270016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017005.973946:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.973950:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017005.973952:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.973957:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.973960:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.973963:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017005.973973:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017005.973983:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335270144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017005.973993:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.973996:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017005.973998:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.974001:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.974004:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.974007:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017005.974013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017005.974021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335270272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017005.974030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.974032:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017005.974034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.974037:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.974039:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.974043:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017005.974046:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017005.974054:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335270400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017005.974063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.974066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017005.974067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.974071:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.974073:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.974076:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017005.974080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017005.974087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335270528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017005.974096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.974099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017005.974101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.974104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.974107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.974110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017005.974114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017005.974121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335270656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017005.974129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017005.974132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017005.974133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017005.974136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1270 00000100:00080000:0.0:1734017005.974139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017005.974142:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017005.974146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017005.974154:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335270784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017006.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017006.989953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017007.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017008.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017009.213914:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017009.995949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017010.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017010.981865:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.981879:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017010.981884:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.981898:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.981909:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.981919:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017010.981968:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335270912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017010.982008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.982013:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017010.982015:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.982019:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.982023:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.982027:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017010.982038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017010.982049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335271040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017010.982059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.982062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017010.982064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.982068:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.982071:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.982075:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017010.982080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017010.982090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335271168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017010.982099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.982102:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017010.982104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.982107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.982110:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.982114:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017010.982119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017010.982128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335271296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017010.982138:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.982141:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017010.982143:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.982147:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.982149:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.982153:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017010.982158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017010.982166:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335271424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017010.982174:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.982177:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017010.982179:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.982183:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.982185:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.982189:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017010.982193:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017010.982201:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335271552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017010.982208:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017010.982212:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017010.982214:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017010.982218:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1275 00000100:00080000:0.0:1734017010.982222:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017010.982225:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017010.982229:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017010.982238:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335271680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017011.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017012.213937:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017013.001991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017013.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017014.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017015.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017015.989848:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.989861:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017015.989867:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.989881:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.989893:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.989903:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017015.989951:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335271808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017015.989988:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.989992:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017015.989994:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.990000:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.990003:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.990007:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017015.990019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017015.990031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335271936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017015.990041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.990045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017015.990047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.990051:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.990053:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.990057:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017015.990062:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017015.990071:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335272064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017015.990081:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.990084:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017015.990086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.990090:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.990092:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.990096:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017015.990100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017015.990109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335272192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017015.990117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.990121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017015.990122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.990127:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.990129:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.990133:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017015.990137:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017015.990145:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335272320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017015.990153:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.990156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017015.990157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.990161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.990164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.990168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017015.990172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017015.990179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335272448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017015.990187:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017015.990190:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017015.990191:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017015.990195:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1280 00000100:00080000:0.0:1734017015.990197:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017015.990200:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017015.990205:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017015.990243:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335272576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017016.007975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017016.213924:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017017.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017018.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:2.0:1734017018.995992:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734017018.996052:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734017018.996076:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017018.996118:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017018.996128:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017018.996137:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017018.996144:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017018.996152:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017018.996203:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00010000:00010000:2.0:1734017019.013950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017019.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017020.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017020.997845:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.997856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017020.997862:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.997875:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.997884:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.997894:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017020.997940:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335272704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017020.997977:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.997982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017020.997984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.997988:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.997992:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.997995:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017020.998006:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017020.998018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335272832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017020.998028:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.998031:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017020.998033:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.998038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.998040:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.998044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017020.998049:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017020.998060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335272960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017020.998069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.998072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017020.998073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.998077:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.998079:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.998083:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017020.998087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017020.998095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335273088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017020.998104:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.998107:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017020.998109:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.998112:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.998115:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.998119:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017020.998123:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017020.998169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335273216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017020.998179:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.998182:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017020.998184:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.998188:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.998191:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.998194:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017020.998199:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017020.998207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335273344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017020.998216:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017020.998219:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017020.998220:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017020.998224:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1285 00000100:00080000:0.0:1734017020.998227:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017020.998231:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017020.998235:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017020.998243:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335273472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017021.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017022.019996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017022.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017023.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017024.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017025.025949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017025.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017026.005825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.005835:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017026.005839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.005850:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.005859:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.005867:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017026.005903:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335273600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017026.005936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.005940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017026.005942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.005946:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.005950:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.005953:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017026.005963:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017026.005973:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335273728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017026.005983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.005986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017026.005988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.005991:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.005994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.005998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017026.006002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017026.006009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335273856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017026.006019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.006022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017026.006023:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.006027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.006029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.006032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017026.006037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017026.006079:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335273984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017026.006089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.006092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017026.006094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.006097:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.006100:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.006104:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017026.006108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017026.006116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335274112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017026.006124:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.006127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017026.006128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.006132:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.006136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.006139:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017026.006143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017026.006151:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335274240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017026.006159:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017026.006162:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017026.006163:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017026.006167:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1290 00000100:00080000:0.0:1734017026.006169:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017026.006172:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017026.006176:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017026.006186:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335274368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017026.213928:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017027.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017028.031995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017028.213956:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017029.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017030.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017031.013827:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.013836:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017031.013841:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.013851:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.013861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.013868:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017031.013904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335274496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017031.013944:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.013948:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017031.013950:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.013992:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.013996:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.014000:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017031.014011:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017031.014023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335274624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017031.014033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.014037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017031.014038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.014042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.014045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.014048:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017031.014053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017031.014062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335274752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017031.014071:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.014074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017031.014076:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.014079:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.014082:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.014086:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017031.014090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017031.014098:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335274880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017031.014105:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.014108:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017031.014110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.014114:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.014116:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.014120:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017031.014124:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017031.014131:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335275008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017031.014138:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.014141:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017031.014143:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.014146:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.014149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.014152:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017031.014156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017031.014164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335275136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017031.014172:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017031.014175:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017031.014177:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017031.014180:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1295 00000100:00080000:0.0:1734017031.014182:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017031.014186:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017031.014189:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017031.014197:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335275264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017031.037948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017031.213919:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017032.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017033.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017034.044004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017034.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017035.213994:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017036.021828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.021839:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017036.021843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.021853:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.021863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.021891:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017036.021927:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335275392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017036.021960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.021964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017036.021966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.021971:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.021974:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.021978:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017036.021987:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017036.021998:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335275520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017036.022007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.022011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017036.022012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.022016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.022019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.022022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017036.022027:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017036.022034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335275648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017036.022042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.022045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017036.022047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.022050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.022053:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.022056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017036.022060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017036.022068:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335275776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017036.022076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.022080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017036.022081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.022085:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.022087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.022090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017036.022094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017036.022101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335275904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017036.022109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.022112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017036.022113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.022117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.022119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.022122:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017036.022126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017036.022135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335276032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017036.022143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017036.022146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017036.022147:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017036.022151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1300 00000100:00080000:0.0:1734017036.022153:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017036.022156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017036.022160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017036.022167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335276160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017036.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017037.050004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017037.213779:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017038.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017039.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017040.055950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017040.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017041.029874:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.029883:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017041.029887:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.029897:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.029945:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.029953:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017041.029990:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335276288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017041.030022:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.030026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017041.030028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.030032:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.030035:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.030039:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017041.030048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017041.030059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335276416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017041.030068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.030071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017041.030074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.030078:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.030080:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.030083:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017041.030088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017041.030097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335276544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017041.030106:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.030108:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017041.030110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.030113:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.030116:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.030119:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017041.030123:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017041.030130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335276672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017041.030139:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.030142:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017041.030144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.030163:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.030166:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.030170:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017041.030175:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017041.030183:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335276800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017041.030192:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.030195:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017041.030196:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.030199:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.030202:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.030205:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017041.030209:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017041.030216:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335276928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017041.030224:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017041.030227:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017041.030228:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017041.030232:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1305 00000100:00080000:0.0:1734017041.030234:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017041.030238:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017041.030242:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017041.030250:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335277056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017041.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017042.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017043.061986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017043.213796:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017044.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017045.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017046.037863:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.037873:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017046.037877:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.037888:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.037898:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.037904:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017046.037938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335277184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017046.037971:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.037976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017046.037977:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.037982:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.037985:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.037989:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017046.037998:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017046.038009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335277312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017046.038018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.038021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017046.038023:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.038027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.038030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.038033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017046.038038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017046.038045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335277440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017046.038055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.038058:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017046.038059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.038063:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.038065:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.038068:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017046.038072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017046.038080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335277568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017046.038089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.038092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017046.038093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.038097:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.038099:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.038103:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017046.038106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017046.038114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335277696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017046.038121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.038124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017046.038125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.038129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.038132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.038136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017046.038140:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017046.038147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335277824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017046.038155:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017046.038158:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017046.038159:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017046.038163:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1310 00000100:00080000:0.0:1734017046.038165:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017046.038168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017046.038172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017046.038181:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335277952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017046.067940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017046.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017047.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017048.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017049.073925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017049.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017050.213782:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017051.047524:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.047534:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017051.047540:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.047565:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.047590:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.047602:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017051.047733:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335278080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017051.047827:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.047834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017051.047837:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.047844:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.047892:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.047900:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017051.047941:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017051.048006:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335278208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017051.048074:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.048103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017051.048107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.048133:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.048159:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.048167:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017051.048175:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017051.048235:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335278336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017051.048293:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.048298:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017051.048301:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.048308:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.048331:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.048338:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017051.048345:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017051.048435:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335278464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017051.048535:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.048542:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017051.048559:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.048566:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.048599:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.048606:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017051.048655:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017051.048725:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335278592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017051.048809:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.048815:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017051.048818:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.048825:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.048847:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.048854:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017051.048874:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017051.048910:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335278720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017051.048930:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017051.048959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017051.048976:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017051.048982:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1315 00000100:00080000:2.0:1734017051.048988:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017051.048993:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017051.049028:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017051.049046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335278848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017051.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017052.079894:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017052.213786:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017053.213785:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017054.213776:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017055.085918:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017055.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017056.053809:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.053817:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017056.053822:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.053832:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.053841:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.053849:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017056.053885:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335278976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017056.053917:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.053921:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017056.053923:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.053928:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.053931:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.053934:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017056.053943:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017056.053954:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335279104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017056.053963:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.053967:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017056.053968:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.053972:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.053976:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.053979:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017056.053983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017056.053992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335279232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017056.054000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.054003:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017056.054004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.054008:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.054010:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.054014:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017056.054017:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017056.054027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335279360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017056.054035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.054039:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017056.054040:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.054044:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.054046:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.054049:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017056.054053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017056.054061:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335279488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017056.054075:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.054078:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017056.054079:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.054083:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.054094:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.054098:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017056.054101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017056.054109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335279616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017056.054117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017056.054120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017056.054122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017056.054125:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1320 00000100:00080000:2.0:1734017056.054128:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017056.054131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017056.054135:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017056.054143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335279744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017056.213882:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017057.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017058.091956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017058.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017059.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017060.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017061.061811:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.061820:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017061.061825:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.061835:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.061844:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.061852:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017061.061886:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335279872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017061.061920:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.061924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017061.061926:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.061930:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.061934:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.061937:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017061.061946:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017061.061957:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335280000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017061.061966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.061969:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017061.061971:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.061974:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.061977:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.061981:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017061.061985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017061.061993:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335280128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017061.062001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.062007:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017061.062009:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.062012:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.062015:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.062018:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017061.062022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017061.062031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335280256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017061.062040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.062043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017061.062044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.062048:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.062050:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.062053:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017061.062057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017061.062064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335280384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017061.062072:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.062075:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017061.062077:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.062080:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.062092:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.062095:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017061.062100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017061.062108:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335280512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017061.062116:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017061.062119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017061.062121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017061.062124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1325 00000100:00080000:2.0:1734017061.062127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017061.062130:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017061.062133:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017061.062141:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335280640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017061.097969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017061.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017062.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017063.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017064.103959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017064.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017065.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017066.069850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.069860:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017066.069864:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.069875:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.069884:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.069891:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017066.069926:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335280768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017066.069959:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.069964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017066.069966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.069971:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.069974:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.069977:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017066.069987:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017066.069998:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335280896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017066.070007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.070011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017066.070012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.070016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.070019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.070022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017066.070026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017066.070034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335281024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017066.070042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.070045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017066.070047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.070050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.070052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.070056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017066.070059:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017066.070069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335281152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017066.070079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.070082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017066.070083:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.070087:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.070089:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.070092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017066.070096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017066.070104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335281280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017066.070112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.070114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017066.070116:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.070119:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.070133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.070137:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017066.070141:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017066.070148:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335281408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017066.070157:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017066.070159:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017066.070161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017066.070165:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1330 00000100:00080000:2.0:1734017066.070168:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017066.070172:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017066.070176:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017066.070183:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335281536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017066.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017067.109947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017067.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017068.213923:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017069.213946:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017070.115966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017070.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017071.077846:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.077855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017071.077860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.077871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.077880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.077887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017071.077923:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335281664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017071.077956:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.077960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017071.077962:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.077967:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.077970:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.077974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017071.077983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017071.077993:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335281792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017071.078011:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.078014:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017071.078016:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.078019:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.078021:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.078025:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017071.078029:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017071.078037:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335281920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017071.078045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.078048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017071.078050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.078053:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.078055:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.078059:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017071.078062:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017071.078072:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335282048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017071.078081:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.078084:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017071.078085:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.078088:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.078092:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.078095:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017071.078099:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017071.078107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335282176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017071.078114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.078117:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017071.078119:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.078122:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.078125:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.078128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017071.078132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017071.078139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335282304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017071.078147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017071.078149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017071.078151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017071.078154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1335 00000100:00080000:2.0:1734017071.078156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017071.078160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017071.078164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017071.078171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335282432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017071.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017072.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017073.121910:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017073.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017074.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017075.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017076.085837:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.085846:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017076.085850:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.085860:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.085870:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.085877:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017076.085913:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335282560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017076.085949:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.085953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017076.085955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.085959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.085962:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.085966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017076.085976:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017076.085986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335282688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017076.085995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.085998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017076.086000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.086004:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.086006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.086010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017076.086015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017076.086023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335282816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017076.086031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.086034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017076.086036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.086040:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.086042:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.086046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017076.086050:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017076.086059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335282944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017076.086067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.086070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017076.086072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.086075:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.086078:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.086081:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017076.086085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017076.086093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335283072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017076.086103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.086106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017076.086108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.086111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.086114:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.086117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017076.086121:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017076.086128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335283200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017076.086136:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017076.086138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017076.086140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017076.086143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1340 00000100:00080000:2.0:1734017076.086146:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017076.086149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017076.086153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017076.086160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335283328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017076.127953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017076.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017077.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017078.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017079.133980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017079.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017080.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017081.093852:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.093862:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017081.093866:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.093876:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.093886:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.093895:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017081.093930:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335283456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017081.093964:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.093968:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017081.093970:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.093975:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.093978:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.093981:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017081.093991:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017081.094001:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335283584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017081.094011:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.094014:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017081.094015:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.094019:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.094022:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.094025:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017081.094029:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017081.094037:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335283712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017081.094046:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.094049:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017081.094051:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.094055:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.094058:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.094061:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017081.094065:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017081.094075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335283840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017081.094083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.094086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017081.094087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.094091:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.094093:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.094096:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017081.094100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017081.094143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335283968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017081.094153:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.094156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017081.094158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.094161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.094164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.094168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017081.094172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017081.094179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335284096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017081.094188:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017081.094191:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017081.094192:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017081.094196:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1345 00000100:00080000:2.0:1734017081.094198:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017081.094201:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017081.094205:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017081.094212:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335284224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017081.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017082.139920:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017082.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017083.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:2.0:1734017084.099967:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734017084.100017:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734017084.100038:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017084.100112:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017084.100122:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017084.100130:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017084.100136:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017084.100143:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017084.100151:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734017084.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017085.145944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017085.213919:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017086.101828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.101837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017086.101842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.101852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.101863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.101870:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017086.101906:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335284352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017086.101942:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.101947:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017086.101949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.101953:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.101956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.101960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017086.101969:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017086.101980:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335284480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017086.101990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.101993:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017086.101994:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.101998:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.102001:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.102004:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017086.102008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017086.102049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335284608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017086.102059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.102064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017086.102066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.102070:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.102073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.102076:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017086.102080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017086.102090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335284736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017086.102098:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.102101:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017086.102103:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.102106:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.102109:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.102112:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017086.102116:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017086.102124:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335284864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017086.102133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.102136:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017086.102138:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.102141:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.102144:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.102147:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017086.102152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017086.102160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335284992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017086.102168:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017086.102171:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017086.102172:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017086.102176:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1350 00000100:00080000:2.0:1734017086.102178:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017086.102181:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017086.102185:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017086.102192:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335285120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017086.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017087.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017088.151912:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017088.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017089.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017090.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017091.109844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.109853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017091.109858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.109868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.109877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.109884:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017091.109918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335285248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017091.109994:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.109998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017091.110000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.110005:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.110009:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.110013:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017091.110023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017091.110035:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335285376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017091.110045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.110048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017091.110050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.110053:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.110056:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.110059:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017091.110064:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017091.110072:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335285504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017091.110081:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.110084:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017091.110086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.110089:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.110091:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.110095:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017091.110099:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017091.110108:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335285632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017091.110118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.110121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017091.110123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.110126:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.110128:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.110132:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017091.110136:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017091.110143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335285760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017091.110151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.110154:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017091.110156:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.110159:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.110162:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.110165:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017091.110169:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017091.110176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335285888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017091.110184:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017091.110188:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017091.110189:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017091.110192:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1355 00000100:00080000:2.0:1734017091.110195:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017091.110198:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017091.110202:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017091.110209:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335286016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017091.157983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017091.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017092.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017093.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017094.163968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017094.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017095.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017096.117808:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.117818:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017096.117822:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.117833:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.117842:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.117875:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017096.117922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335286144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017096.117955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.117960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017096.117962:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.117967:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.117970:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.117974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017096.117983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017096.117994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335286272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017096.118004:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.118008:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017096.118009:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.118013:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.118016:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.118019:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017096.118023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017096.118030:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335286400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017096.118040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.118043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017096.118045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.118048:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.118051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.118054:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017096.118058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017096.118068:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335286528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017096.118077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.118080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017096.118081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.118085:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.118087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.118091:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017096.118094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017096.118101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335286656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017096.118109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.118112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017096.118113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.118117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.118119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.118123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017096.118127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017096.118145:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335286784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017096.118153:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017096.118156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017096.118157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017096.118161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1360 00000100:00080000:2.0:1734017096.118164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017096.118168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017096.118172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017096.118179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335286912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017096.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017097.169978:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017097.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017098.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017099.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017100.175976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017100.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017101.125846:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.125855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017101.125860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.125870:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.125880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.125887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017101.125922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335287040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017101.125955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.125959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017101.125961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.125965:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.125969:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.125972:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017101.125984:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017101.125994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335287168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017101.126004:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.126008:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017101.126009:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.126013:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.126015:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.126019:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017101.126023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017101.126031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335287296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017101.126039:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.126042:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017101.126043:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.126047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.126050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.126053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017101.126057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017101.126066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335287424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017101.126074:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.126077:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017101.126079:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.126082:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.126085:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.126088:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017101.126092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017101.126099:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335287552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017101.126108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.126111:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017101.126113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.126116:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.126118:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.126122:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017101.126126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017101.126133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335287680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017101.126141:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017101.126143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017101.126145:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017101.126148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1365 00000100:00080000:2.0:1734017101.126153:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017101.126156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017101.126160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017101.126167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335287808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017101.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017102.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017103.181956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017103.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017104.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017105.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017106.133777:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.133784:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017106.133787:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.133794:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.133801:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.133806:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017106.133832:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335287936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017106.133864:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.133867:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017106.133868:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.133871:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.133872:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.133875:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017106.133881:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017106.133887:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335288064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017106.133893:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.133896:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017106.133897:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.133899:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.133901:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.133903:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017106.133906:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017106.133929:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335288192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017106.133938:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.133941:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017106.133942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.133956:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.133959:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.133962:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017106.133967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017106.133976:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335288320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017106.133984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.133987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017106.133989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.133992:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.133994:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.133997:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017106.134002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017106.134010:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335288448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017106.134028:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.134031:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017106.134032:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.134035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.134037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.134041:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017106.134044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017106.134051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335288576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017106.134061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017106.134063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017106.134065:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017106.134068:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1370 00000100:00080000:2.0:1734017106.134070:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017106.134074:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017106.134078:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017106.134086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335288704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017106.187846:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017106.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017107.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017108.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017109.193950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017109.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017110.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017111.141814:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.141825:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017111.141831:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.141844:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.141855:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.141864:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017111.141912:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335288832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017111.141951:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.141955:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017111.141957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.141962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.141966:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.141970:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017111.141981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017111.141992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335288960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017111.142002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.142005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017111.142007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.142011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.142014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.142018:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017111.142022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017111.142031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335289088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017111.142041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.142044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017111.142046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.142050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.142052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.142056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017111.142060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017111.142071:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335289216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017111.142079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.142083:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017111.142084:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.142088:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.142091:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.142094:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017111.142098:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017111.142106:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335289344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017111.142114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.142118:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017111.142120:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.142124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.142142:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.142146:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017111.142151:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017111.142159:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335289472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017111.142167:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017111.142170:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017111.142172:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017111.142175:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1375 00000100:00080000:2.0:1734017111.142179:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017111.142183:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017111.142187:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017111.142195:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335289600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017111.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017112.199944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017112.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017113.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017114.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017115.205963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017115.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017116.149825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.149837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017116.149843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.149856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.149866:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.149875:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017116.149922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335289728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017116.149966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.149971:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017116.149973:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.149978:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.149981:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.149985:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017116.149996:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017116.150007:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335289856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017116.150017:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.150020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017116.150022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.150026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.150029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.150032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017116.150037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017116.150045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335289984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017116.150053:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.150057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017116.150058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.150062:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.150064:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.150068:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017116.150073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017116.150085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335290112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017116.150093:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.150097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017116.150098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.150102:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.150105:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.150108:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017116.150113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017116.150121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335290240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017116.150129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.150132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017116.150133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.150137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.150149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.150152:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017116.150156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017116.150165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335290368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017116.150173:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017116.150177:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017116.150178:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017116.150182:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1380 00000100:00080000:2.0:1734017116.150185:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017116.150189:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017116.150193:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017116.150201:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335290496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017116.213937:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017117.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017118.211947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017118.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017119.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017120.213956:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017121.157941:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.157952:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017121.157957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.157970:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.157980:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.157989:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017121.158035:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335290624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017121.158077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.158082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017121.158084:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.158089:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.158092:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.158096:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017121.158107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017121.158119:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335290752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017121.158129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.158133:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017121.158135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.158139:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.158141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.158145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017121.158151:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017121.158162:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335290880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017121.158171:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.158174:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017121.158176:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.158179:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.158182:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.158185:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017121.158189:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017121.158198:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335291008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017121.158206:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.158210:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017121.158211:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.158215:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.158218:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.158221:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017121.158226:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017121.158234:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335291136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017121.158242:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.158245:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017121.158248:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.158252:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.158255:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.158258:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017121.158263:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017121.158270:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335291264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017121.158278:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017121.158281:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017121.158282:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017121.158286:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1385 00000100:00080000:0.0:1734017121.158289:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017121.158292:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017121.158296:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017121.158305:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335291392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017121.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017121.217980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017122.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017123.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017124.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017124.223928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017125.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017126.165831:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.165841:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017126.165846:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.165856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.165866:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.165874:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017126.165910:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335291520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017126.165945:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.165950:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017126.165952:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.165956:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.165959:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.165963:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017126.165972:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017126.165983:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335291648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017126.165992:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.165996:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017126.165997:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.166001:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.166003:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.166007:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017126.166011:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017126.166019:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335291776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017126.166027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.166033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017126.166034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.166038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.166040:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.166043:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017126.166047:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017126.166055:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335291904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017126.166063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.166067:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017126.166068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.166071:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.166074:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.166077:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017126.166081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017126.166087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335292032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017126.166095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.166098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017126.166100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.166103:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.166106:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.166109:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017126.166114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017126.166121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335292160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017126.166129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017126.166132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017126.166134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017126.166137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1390 00000100:00080000:0.0:1734017126.166140:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017126.166143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017126.166147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017126.166156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335292288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017126.213800:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017127.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017127.229923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017128.213783:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017129.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017130.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017130.235953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734017131.173812:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.173821:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017131.173825:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.173836:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.173845:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.173852:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017131.173889:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335292416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017131.173930:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.173934:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017131.173936:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.173940:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.173943:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.173947:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017131.173956:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017131.173967:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335292544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017131.173976:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.173979:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017131.173981:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.173984:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.173987:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.173991:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017131.173995:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017131.174005:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335292672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017131.174013:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.174016:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017131.174018:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.174021:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.174023:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.174028:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017131.174032:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017131.174041:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335292800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017131.174049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.174053:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017131.174054:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.174057:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.174060:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.174063:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017131.174067:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017131.174074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335292928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017131.174082:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.174085:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017131.174086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.174090:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.174092:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.174096:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017131.174100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017131.174107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335293056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017131.174115:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017131.174118:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017131.174120:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017131.174123:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1395 00000100:00080000:0.0:1734017131.174126:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017131.174129:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017131.174133:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017131.174141:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335293184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017131.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017132.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017133.213799:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017133.241886:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017134.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017135.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017136.181840:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.181853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017136.181859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.181873:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.181884:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.181894:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017136.181942:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335293312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017136.181982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.181987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017136.181989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.181994:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.181999:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.182003:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017136.182014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017136.182026:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335293440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017136.182036:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.182040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017136.182042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.182046:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.182048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.182052:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017136.182056:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017136.182065:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335293568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017136.182073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.182077:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017136.182078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.182082:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.182085:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.182088:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017136.182092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017136.182100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335293696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017136.182116:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.182120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017136.182121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.182125:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.182128:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.182131:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017136.182136:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017136.182144:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335293824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017136.182152:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.182155:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017136.182156:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.182160:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.182162:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.182197:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017136.182202:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017136.182212:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335293952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017136.182221:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017136.182224:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017136.182226:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017136.182230:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1400 00000100:00080000:0.0:1734017136.182235:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017136.182238:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017136.182243:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017136.182253:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335294080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017136.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017136.247968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017137.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017138.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017139.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017139.253873:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017140.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017141.189841:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.189854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017141.189860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.189875:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.189886:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.189895:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017141.189945:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335294208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017141.189985:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.189990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017141.189992:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.189997:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.190000:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.190004:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017141.190016:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017141.190030:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335294336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017141.190040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.190044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017141.190046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.190049:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.190052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.190056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017141.190060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017141.190072:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335294464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017141.190117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.190120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017141.190122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.190126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.190129:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.190133:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017141.190138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017141.190148:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335294592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017141.190159:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.190163:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017141.190164:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.190169:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.190172:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.190176:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017141.190181:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017141.190189:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335294720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017141.190197:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.190200:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017141.190201:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.190205:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.190207:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.190211:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017141.190216:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017141.190224:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335294848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017141.190232:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017141.190235:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017141.190236:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017141.190240:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1405 00000100:00080000:0.0:1734017141.190244:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017141.190247:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017141.190251:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017141.190259:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335294976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017141.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017142.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017142.259958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017143.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017144.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017145.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017145.265941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734017146.197844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.197854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017146.197859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.197869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.197879:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.197887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017146.197924:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335295104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017146.197957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.197961:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017146.197963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.197967:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.197970:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.197974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017146.197983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017146.198028:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335295232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017146.198042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.198045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017146.198047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.198051:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.198055:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.198059:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017146.198065:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017146.198076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335295360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017146.198086:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.198089:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017146.198091:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.198094:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.198097:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.198100:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017146.198104:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017146.198112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335295488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017146.198120:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.198123:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017146.198125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.198128:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.198131:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.198134:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017146.198138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017146.198145:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335295616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017146.198154:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.198157:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017146.198158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.198162:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.198164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.198168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017146.198172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017146.198179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335295744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017146.198186:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017146.198190:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017146.198191:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017146.198194:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1410 00000100:00080000:0.0:1734017146.198197:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017146.198200:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017146.198204:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017146.198213:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335295872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017146.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017147.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017148.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017148.271949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1734017149.204019:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734017149.204065:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734017149.204085:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017149.204122:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017149.204132:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017149.204140:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017149.204147:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017149.204154:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017149.204160:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734017149.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017150.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017151.205969:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.205978:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017151.205983:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.205994:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206003:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206011:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017151.206044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335296000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017151.206080:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.206085:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017151.206086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.206091:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206094:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206097:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017151.206107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017151.206117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335296128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017151.206127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.206130:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017151.206132:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.206136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017151.206147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017151.206157:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335296256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017151.206165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.206168:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017151.206170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.206173:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206176:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206179:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017151.206183:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017151.206191:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335296384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017151.206200:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.206203:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017151.206205:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.206208:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206213:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206216:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017151.206220:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017151.206228:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335296512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017151.206242:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.206245:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017151.206247:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.206250:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206253:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206256:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017151.206260:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017151.206267:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335296640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017151.206275:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017151.206277:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017151.206279:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017151.206282:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1415 00000100:00080000:2.0:1734017151.206285:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017151.206288:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017151.206292:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017151.206299:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335296768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017151.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017151.277977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017152.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017153.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017154.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017154.283942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017155.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017156.213860:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734017156.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017156.213871:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017156.213876:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.213887:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.213897:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.213904:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017156.213940:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335296896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017156.213973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017156.213977:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017156.213979:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.213983:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.213986:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.213990:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017156.213999:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017156.214009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335297024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017156.214019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017156.214022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017156.214024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.214027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.214030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.214033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017156.214038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017156.214045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335297152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017156.214054:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017156.214057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017156.214059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.214063:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.214065:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.214068:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017156.214072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017156.214082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335297280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017156.214091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017156.214094:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017156.214095:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.214099:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.214101:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.214104:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017156.214108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017156.214116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335297408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017156.214123:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017156.214126:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017156.214128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.214131:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.214148:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.214152:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017156.214156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017156.214164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335297536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017156.214172:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017156.214175:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017156.214177:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017156.214180:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1420 00000100:00080000:2.0:1734017156.214183:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017156.214186:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017156.214190:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017156.214197:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335297664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017157.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017157.289954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017158.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017159.213890:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017160.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017160.295979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017161.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017161.221844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.221853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017161.221859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.221871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.221882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.221892:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017161.221939:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335297792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017161.221978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.221982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017161.221985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.221990:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.221993:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.221997:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017161.222009:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017161.222020:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335297920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017161.222030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.222034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017161.222036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.222040:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.222043:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.222046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017161.222051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017161.222059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335298048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017161.222068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.222071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017161.222072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.222076:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.222079:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.222082:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017161.222086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017161.222097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335298176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017161.222107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.222110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017161.222112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.222116:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.222118:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.222122:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017161.222126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017161.222135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335298304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017161.222143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.222146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017161.222147:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.222151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.222153:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.222157:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017161.222161:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017161.222169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335298432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017161.222176:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017161.222179:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017161.222181:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017161.222184:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1425 00000100:00080000:2.0:1734017161.222188:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017161.222192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017161.222197:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017161.222205:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335298560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017162.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017163.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017163.301943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017164.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017165.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017166.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017166.229804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.229812:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017166.229817:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.229827:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.229836:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.229844:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017166.229879:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335298688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017166.229913:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.229917:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017166.229920:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.229924:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.229927:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.229931:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017166.229940:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017166.229950:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335298816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017166.229960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.229963:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017166.229965:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.229969:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.229972:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.229975:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017166.229979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017166.229987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335298944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017166.229996:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.229998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017166.230000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.230003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.230006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.230009:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017166.230013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017166.230023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335299072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017166.230031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.230034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017166.230036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.230039:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.230042:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.230045:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017166.230050:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017166.230057:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335299200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017166.230065:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.230068:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017166.230070:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.230073:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.230085:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.230088:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017166.230092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017166.230100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335299328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017166.230108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017166.230111:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017166.230112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017166.230116:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1430 00000100:00080000:2.0:1734017166.230119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017166.230122:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017166.230126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017166.230133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335299456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00010000:00010000:2.0:1734017166.307860:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017167.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017168.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017169.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017169.313925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017170.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017171.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017171.237853:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.237862:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017171.237866:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.237876:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.237886:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.237894:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017171.237928:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335299584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017171.237959:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.237963:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017171.237965:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.237969:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.237972:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.237976:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017171.237985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017171.237995:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335299712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017171.238004:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.238007:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017171.238009:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.238013:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.238016:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.238019:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017171.238023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017171.238031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335299840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017171.238040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.238043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017171.238044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.238048:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.238050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.238053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017171.238058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017171.238066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335299968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017171.238075:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.238078:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017171.238079:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.238083:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.238087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.238090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017171.238094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017171.238101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335300096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017171.238109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.238112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017171.238114:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.238117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.238120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.238124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017171.238128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017171.238135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335300224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017171.238142:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017171.238145:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017171.238146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017171.238150:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1435 00000100:00080000:2.0:1734017171.238160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017171.238163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017171.238167:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017171.238175:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335300352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017172.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734017172.319906:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017173.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017174.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017175.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017176.213920:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017176.245836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.245844:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017176.245849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.245858:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.245868:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.245876:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017176.245913:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335300480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017176.245954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.245958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017176.245959:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.245963:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.245967:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.245970:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017176.245979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017176.245989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335300608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017176.245999:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.246002:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017176.246003:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.246007:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.246010:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.246013:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017176.246018:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017176.246025:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335300736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017176.246034:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.246037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017176.246038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.246042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.246044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.246048:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017176.246052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017176.246062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335300864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017176.246071:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.246074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017176.246076:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.246079:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.246081:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.246085:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017176.246089:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017176.246097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335300992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017176.246105:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.246107:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017176.246109:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.246112:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.246124:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.246128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017176.246132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017176.246139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335301120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017176.246147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017176.246151:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017176.246152:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017176.246156:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1440 00000100:00080000:2.0:1734017176.246159:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017176.246162:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017176.246166:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017176.246173:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335301248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017177.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017178.213928:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017179.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017180.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017181.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017181.253820:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.253828:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017181.253833:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.253842:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.253852:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.253860:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017181.253896:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335301376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017181.253929:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.253934:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017181.253935:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.253940:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.253943:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.253947:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017181.253956:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017181.253968:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335301504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017181.253978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.253982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017181.253984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.253987:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.253990:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.253993:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017181.253998:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017181.254005:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335301632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017181.254013:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.254016:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017181.254018:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.254021:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.254024:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.254027:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017181.254031:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017181.254040:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335301760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017181.254049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.254052:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017181.254054:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.254057:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.254060:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.254063:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017181.254067:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017181.254074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335301888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017181.254082:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.254085:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017181.254086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.254090:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.254092:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.254096:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017181.254099:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017181.254106:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335302016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017181.254114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017181.254117:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017181.254118:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017181.254122:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1445 00000100:00080000:2.0:1734017181.254125:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017181.254128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017181.254132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017181.254139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335302144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017182.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017183.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017184.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017185.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017186.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017186.261831:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.261841:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017186.261845:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.261856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.261865:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.261873:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017186.261909:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335302272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017186.261941:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.261945:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017186.261947:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.261951:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.261954:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.261958:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017186.261967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017186.261978:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335302400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017186.261988:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.261991:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017186.261993:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.261996:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.261999:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.262002:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017186.262007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017186.262015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335302528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017186.262024:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.262027:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017186.262029:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.262032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.262034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.262038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017186.262042:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017186.262052:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335302656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017186.262060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.262063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017186.262065:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.262069:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.262071:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.262075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017186.262079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017186.262086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335302784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017186.262095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.262098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017186.262100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.262104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.262116:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.262119:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017186.262124:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017186.262131:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335302912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017186.262139:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017186.262142:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017186.262144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017186.262147:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1450 00000100:00080000:2.0:1734017186.262150:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017186.262153:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017186.262157:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017186.262164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335303040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017187.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017188.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017189.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017190.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017191.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017191.269842:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.269851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017191.269855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.269865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.269875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.269882:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017191.269917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335303168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017191.269957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.269962:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017191.269963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.269968:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.269971:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.269974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017191.269984:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017191.269994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335303296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017191.270003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.270006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017191.270008:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.270012:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.270014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.270018:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017191.270022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017191.270030:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335303424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017191.270038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.270041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017191.270043:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.270047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.270049:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.270053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017191.270057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017191.270066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335303552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017191.270075:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.270078:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017191.270080:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.270083:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.270087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.270091:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017191.270095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017191.270102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335303680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017191.270111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.270114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017191.270115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.270118:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.270130:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.270134:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017191.270138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017191.270145:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335303808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017191.270152:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017191.270156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017191.270157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017191.270161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1455 00000100:00080000:2.0:1734017191.270164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017191.270167:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017191.270171:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017191.270178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335303936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017192.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017193.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017194.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017195.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017196.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017196.277923:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.277931:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017196.277935:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.277945:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.277954:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.277962:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017196.277997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335304064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017196.278034:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.278038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017196.278040:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.278044:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.278047:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.278051:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017196.278061:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017196.278071:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335304192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017196.278088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.278092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017196.278093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.278097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.278099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.278103:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017196.278107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017196.278116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335304320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017196.278124:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.278127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017196.278128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.278132:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.278134:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.278137:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017196.278169:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017196.278180:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335304448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017196.278190:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.278193:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017196.278195:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.278198:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.278202:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.278206:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017196.278210:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017196.278218:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335304576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017196.278225:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.278229:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017196.278230:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.278233:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.278236:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.278239:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017196.278243:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017196.278251:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335304704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017196.278259:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017196.278261:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017196.278263:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017196.278266:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1460 00000100:00080000:0.0:1734017196.278269:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017196.278272:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017196.278276:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017196.278283:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335304832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017197.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017198.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017199.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017200.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017201.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017201.285849:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.285858:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017201.285862:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.285872:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.285882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.285889:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017201.285926:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335304960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017201.285959:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.285964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017201.285966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.285970:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.285973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.285977:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017201.285986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017201.285997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335305088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017201.286006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.286009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017201.286012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.286016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.286018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.286022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017201.286026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017201.286034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335305216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017201.286042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.286045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017201.286047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.286050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.286052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.286093:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017201.286098:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017201.286107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335305344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017201.286116:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.286120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017201.286121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.286125:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.286127:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.286131:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017201.286135:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017201.286142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335305472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017201.286151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.286154:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017201.286156:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.286159:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.286161:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.286165:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017201.286169:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017201.286176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335305600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017201.286183:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017201.286186:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017201.286188:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017201.286191:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1465 00000100:00080000:0.0:1734017201.286194:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017201.286197:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017201.286201:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017201.286210:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335305728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017202.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017203.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017204.213941:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017205.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017206.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017206.293833:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.293842:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017206.293847:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.293858:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.293868:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.293876:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017206.293912:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335305856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017206.293945:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.293949:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017206.293951:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.293955:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.293958:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.293962:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017206.294013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017206.294027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335305984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017206.294037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.294040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017206.294042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.294046:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.294049:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.294052:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017206.294057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017206.294067:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335306112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017206.294076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.294080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017206.294082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.294086:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.294089:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.294092:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017206.294096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017206.294104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335306240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017206.294112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.294115:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017206.294116:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.294120:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.294122:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.294126:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017206.294129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017206.294136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335306368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017206.294144:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.294147:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017206.294148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.294152:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.294155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.294158:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017206.294163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017206.294171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335306496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017206.294179:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017206.294182:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017206.294183:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017206.294187:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1470 00000100:00080000:0.0:1734017206.294189:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017206.294192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017206.294196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017206.294204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335306624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017207.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017208.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017209.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017210.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017211.213971:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017211.301843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.301851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017211.301856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.301866:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.301876:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.301912:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017211.301949:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335306752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017211.301982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.301986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017211.301988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.301993:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.301996:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.302000:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017211.302009:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017211.302020:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335306880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017211.302030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.302033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017211.302035:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.302038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.302041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.302044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017211.302048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017211.302056:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335307008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017211.302064:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.302067:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017211.302069:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.302072:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.302075:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.302078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017211.302082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017211.302089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335307136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017211.302099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.302103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017211.302104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.302108:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.302110:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.302113:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017211.302117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017211.302124:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335307264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017211.302132:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.302135:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017211.302136:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.302140:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.302142:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.302146:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017211.302150:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017211.302157:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335307392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017211.302165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017211.302168:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017211.302170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017211.302173:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1475 00000100:00080000:0.0:1734017211.302176:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017211.302180:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017211.302186:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017211.302201:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335307520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017212.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017213.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017214.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734017214.307978:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734017214.308037:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734017214.308057:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017214.308091:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017214.308101:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017214.308108:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017214.308116:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017214.308123:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017214.308129:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734017215.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017216.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017216.309863:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.309871:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017216.309876:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.309886:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.309895:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.309902:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017216.309939:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335307648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017216.309969:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.309974:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017216.309976:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.309980:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.309983:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.309987:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017216.309996:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017216.310007:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335307776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017216.310017:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.310020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017216.310021:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.310025:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.310027:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.310031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017216.310035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017216.310045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335307904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017216.310053:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.310056:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017216.310058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.310061:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.310064:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.310067:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017216.310071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017216.310079:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335308032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017216.310088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.310091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017216.310092:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.310097:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.310099:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.310103:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017216.310107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017216.310114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335308160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017216.310121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.310124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017216.310126:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.310129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.310132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.310135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017216.310139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017216.310146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335308288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017216.310154:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017216.310157:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017216.310158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017216.310161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1480 00000100:00080000:0.0:1734017216.310164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017216.310168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017216.310172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017216.310179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335308416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017217.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017218.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017219.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017220.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017221.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017221.317850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.317858:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017221.317863:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.317873:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.317882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.317890:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017221.317927:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335308544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017221.317960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.317964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017221.317966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.317970:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.317973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.317977:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017221.317987:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017221.317997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335308672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017221.318007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.318010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017221.318012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.318016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.318019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.318022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017221.318026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017221.318034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335308800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017221.318042:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.318045:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017221.318047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.318050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.318052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.318055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017221.318059:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017221.318067:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335308928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017221.318075:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.318079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017221.318080:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.318084:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.318086:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.318089:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017221.318093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017221.318100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335309056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017221.318109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.318112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017221.318113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.318117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.318119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.318123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017221.318126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017221.318134:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335309184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017221.318141:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017221.318144:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017221.318146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017221.318149:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1485 00000100:00080000:0.0:1734017221.318152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017221.318155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017221.318158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017221.318167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335309312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017222.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017223.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017224.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017225.213939:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017226.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017226.325854:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.325862:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017226.325867:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.325877:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.325887:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.325895:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017226.325931:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335309440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017226.325960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.325964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017226.325966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.325970:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.325973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.325976:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017226.325986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017226.325996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335309568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017226.326006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.326009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017226.326010:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.326014:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.326017:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.326020:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017226.326024:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017226.326034:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335309696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017226.326049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.326053:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017226.326054:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.326057:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.326060:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.326063:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017226.326067:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017226.326075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335309824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017226.326084:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.326087:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017226.326088:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.326092:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.326094:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.326097:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017226.326101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017226.326108:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335309952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017226.326116:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.326119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017226.326120:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.326123:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.326127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.326130:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017226.326134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017226.326142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335310080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017226.326149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017226.326152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017226.326154:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017226.326157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1490 00000100:00080000:0.0:1734017226.326159:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017226.326163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017226.326166:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017226.326174:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335310208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017227.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017228.213965:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017229.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017230.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017231.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017231.333833:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.333841:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017231.333845:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.333855:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.333864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.333872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017231.333907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335310336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017231.333940:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.333944:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017231.333946:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.333950:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.333953:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.333957:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017231.333966:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017231.333976:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335310464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017231.333986:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.333989:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017231.333991:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.333994:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.333997:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.334001:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017231.334005:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017231.334012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335310592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017231.334021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.334024:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017231.334025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.334028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.334031:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.334035:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017231.334039:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017231.334047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335310720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017231.334055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.334059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017231.334060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.334064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.334066:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.334069:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017231.334073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017231.334080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335310848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017231.334088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.334091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017231.334092:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.334095:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.334098:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.334101:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017231.334105:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017231.334112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335310976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017231.334120:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017231.334124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017231.334125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017231.334129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1495 00000100:00080000:0.0:1734017231.334132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017231.334135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017231.334138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017231.334147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335311104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017232.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017233.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017234.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017235.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017236.213922:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017236.341836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.341844:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017236.341849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.341858:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.341867:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.341874:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017236.341909:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335311232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017236.341942:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.341947:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017236.341949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.341953:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.341956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.341961:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017236.341970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017236.341981:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335311360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017236.341990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.341993:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017236.341995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.341998:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.342001:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.342004:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017236.342008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017236.342018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335311488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017236.342027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.342030:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017236.342031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.342035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.342037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.342041:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017236.342045:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017236.342052:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335311616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017236.342061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.342065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017236.342067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.342070:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.342072:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.342076:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017236.342080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017236.342087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335311744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017236.342095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.342097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017236.342099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.342102:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.342105:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.342108:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017236.342112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017236.342119:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335311872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017236.342127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017236.342130:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017236.342131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017236.342135:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1500 00000100:00080000:0.0:1734017236.342137:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017236.342140:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017236.342145:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017236.342153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335312000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017237.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017238.213929:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017239.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017240.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017241.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017241.349863:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.349871:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017241.349876:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.349887:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.349896:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.349904:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017241.349938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335312128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017241.349973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.349978:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017241.349980:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.349984:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.349988:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.349991:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017241.350001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017241.350011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335312256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017241.350021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.350025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017241.350027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.350031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.350034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.350037:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017241.350041:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017241.350049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335312384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017241.350057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.350060:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017241.350062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.350065:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.350067:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.350071:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017241.350075:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017241.350082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335312512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017241.350091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.350094:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017241.350096:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.350099:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.350101:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.350105:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017241.350110:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017241.350117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335312640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017241.350125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.350128:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017241.350130:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.350133:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.350136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.350139:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017241.350143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017241.350149:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335312768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017241.350157:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017241.350160:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017241.350161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017241.350165:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1505 00000100:00080000:0.0:1734017241.350167:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017241.350170:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017241.350174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017241.350183:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335312896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017242.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017243.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017244.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017245.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017246.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017246.357843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.357851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017246.357855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.357865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.357874:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.357881:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017246.357917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335313024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017246.357949:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.357953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017246.357955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.357959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.357963:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.357966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017246.357977:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017246.357987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335313152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017246.357997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.358000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017246.358001:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.358005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.358008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.358011:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017246.358017:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017246.358027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335313280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017246.358035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.358038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017246.358039:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.358043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.358045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.358049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017246.358052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017246.358060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335313408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017246.358070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.358073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017246.358074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.358078:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.358080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.358084:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017246.358087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017246.358094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335313536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017246.358102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.358105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017246.358107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.358111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.358114:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.358117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017246.358121:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017246.358128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335313664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017246.358136:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017246.358139:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017246.358140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017246.358144:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1510 00000100:00080000:0.0:1734017246.358146:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017246.358149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017246.358153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017246.358160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335313792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017247.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017248.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017249.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017250.213799:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734017251.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017251.365856:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.365864:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017251.365869:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.365878:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.365889:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.365896:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017251.365932:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335313920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017251.365965:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.365969:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017251.365971:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.365976:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.365979:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.365983:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017251.365992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017251.366003:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335314048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017251.366013:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.366016:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017251.366018:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.366022:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.366024:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.366027:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017251.366032:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017251.366039:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335314176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017251.366047:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.366053:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017251.366055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.366058:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.366061:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.366064:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017251.366068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017251.366076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335314304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017251.366084:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.366088:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017251.366089:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.366092:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.366095:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.366098:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017251.366102:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017251.366109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335314432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017251.366117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.366119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017251.366121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.366124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.366127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.366130:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017251.366135:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017251.366143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335314560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017251.366151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017251.366154:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017251.366155:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017251.366158:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1515 00000100:00080000:0.0:1734017251.366161:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017251.366164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017251.366168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017251.366177:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335314688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017251.366546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335314688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017251.366561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017251.366563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017251.366566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335314688:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017251.477938:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017252.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017252.479953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017253.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017253.481972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017254.213920:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017254.483971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017255.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017255.485963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017256.213923:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017256.373838:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.373846:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017256.373850:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.373860:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.373872:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017256.373879:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017256.373915:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335314816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017256.373946:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.373953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017256.373955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.373959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.373962:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017256.373966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017256.373975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017256.373986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335314944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017256.373996:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.373999:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017256.374000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.374004:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.374006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017256.374010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017256.374014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017256.374024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335315072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017256.374032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.374035:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017256.374036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.374040:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.374043:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017256.374046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017256.374051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017256.374055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335314816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017256.374059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335315200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017256.374068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.374071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017256.374072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.374076:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.374078:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017256.374081:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017256.374085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017256.374092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335315328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017256.374100:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.374103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017256.374104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.374107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.374110:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017256.374113:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017256.374155:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734017256.374163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017256.374165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335315456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017256.374174:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017256.374177:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017256.374180:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017256.374183:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1520 00000100:00080000:0.0:1734017256.374187:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734017256.374187:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59c8000 x1818247335314816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017256.374190:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017256.374195:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017256.374202:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335315584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017256.374206:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335314816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374220:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374224:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335314816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017256.374232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335314944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017256.374252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017256.374259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59c9500 x1818247335314944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017256.374271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335314944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335314944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017256.374298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017256.374312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017256.374318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59ca300 x1818247335315072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017256.374328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335315072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017256.374366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017256.374378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017256.374384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59c9180 x1818247335315200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017256.374394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335315200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017256.374412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335315328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017256.374424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017256.374430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59cb100 x1818247335315328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017256.374440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335315328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335315328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017256.374458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335315456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017256.374469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017256.374475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59cb800 x1818247335315456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017256.374484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335315456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335315456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017256.374502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335315584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017256.374513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017256.374519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017256/real 1734017256] req@ffff8800b59cad80 x1818247335315584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017256.374530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335315584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017256.374537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017256.374539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017256.374542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335315584:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017256.487939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017257.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017257.489985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017258.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017258.492008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017259.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017259.493963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017260.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017260.495950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017261.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017261.381806:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.381814:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017261.381819:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.381829:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.381838:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017261.381846:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017261.381882:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335315712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017261.381914:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.381918:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017261.381920:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.381925:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.381928:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017261.381931:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017261.381942:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017261.381953:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335315840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017261.381963:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.381966:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017261.381968:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.381971:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.381974:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017261.381977:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017261.381982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017261.381989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335315968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017261.381998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.382001:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017261.382002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.382006:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.382008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017261.382012:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017261.382016:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017261.382018:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335315712:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017261.382023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335316096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017261.382061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.382065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017261.382067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.382072:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.382075:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017261.382078:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017261.382083:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017261.382092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335316224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017261.382100:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.382104:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017261.382106:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.382109:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.382112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017261.382115:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017261.382119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734017261.382121:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017261.382127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335316352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017261.382135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017261.382137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017261.382139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017261.382142:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1525 00000100:00080000:0.0:1734017261.382144:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734017261.382144:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59cb480 x1818247335315712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017261.382148:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017261.382152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017261.382161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335316480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017261.382163:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335315712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382173:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382177:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382181:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335315712:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017261.382189:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017261.382206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017261.382213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59ca680 x1818247335315840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017261.382232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335315840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017261.382252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017261.382265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017261.382271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59c8e00 x1818247335315968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017261.382281:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335315968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335315968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017261.382319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017261.382332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017261.382338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59caa00 x1818247335316096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017261.382349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335316096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017261.382367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335316224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017261.382378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017261.382384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59c8a80 x1818247335316224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017261.382394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335316224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335316224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017261.382411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335316352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017261.382422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017261.382428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59c8700 x1818247335316352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017261.382438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335316352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335316352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017261.382455:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335316480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017261.382470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017261.382476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017261/real 1734017261] req@ffff8800b59c9f80 x1818247335316480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017261.382487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335316480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017261.382494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017261.382495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017261.382498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335316480:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017261.497930:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017262.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017262.499933:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017263.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017263.501927:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017264.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017264.503942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017265.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017265.505945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017266.213929:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017266.389830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017266.389840:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017266.389844:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.389854:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00080000:0.0:1734017266.389863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.389870:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017266.389905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335316608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017266.389938:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017266.389943:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017266.389944:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.389985:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00080000:0.0:1734017266.389989:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.389993:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017266.390004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017266.390016:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335316736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017266.390026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017266.390029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017266.390031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.390035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00080000:0.0:1734017266.390038:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.390041:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017266.390046:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:3.0:1734017266.390047:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335316608:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017266.390055:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335316864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017266.390064:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017266.390067:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017266.390069:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.390072:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00080000:0.0:1734017266.390075:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.390078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017266.390082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017266.390090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335316992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017266.390098:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017266.390101:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017266.390102:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.390106:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00080000:0.0:1734017266.390108:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.390112:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734017266.390113:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017266.390116:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017266.390122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335317120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017266.390131:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017266.390134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1734017266.390134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59c9c00 x1818247335316608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017266.390136:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.390139:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00080000:0.0:1734017266.390142:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.390145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017266.390149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017266.390153:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335316608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017266.390156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335317248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017266.390164:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1734017266.390164:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017266.390167:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1734017266.390167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734017266.390168:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017266.390172:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1530 00000100:00100000:3.0:1734017266.390172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335316608:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017266.390174:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017266.390177:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390180:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316736:192.168.201.104@tcp:38:kworker.0 00000008:00100000:0.0:1734017266.390181:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017266.390188:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335317376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734017266.390200:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59cbb80 x1818247335316736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017266.390218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335316736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017266.390226:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017266.390229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017266.390232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017266.390262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017266.390275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59c9f80 x1818247335316864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017266.390291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335316864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017266.390298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017266.390300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017266.390304:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017266.390309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017266.390321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390326:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59c8700 x1818247335316992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017266.390343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335316992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017266.390351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017266.390353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017266.390356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335316992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017266.390361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335317120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017266.390373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59c8a80 x1818247335317120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017266.390389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335317120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017266.390396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017266.390398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017266.390401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335317120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017266.390406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335317248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017266.390418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59caa00 x1818247335317248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017266.390434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335317248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017266.390442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017266.390444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017266.390448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335317248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017266.390454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335317376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017266.390466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017266.390471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017266/real 1734017266] req@ffff8800b59c8e00 x1818247335317376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017266.390481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335317376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017266.390488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017266.390490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017266.390493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335317376:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017266.507853:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017267.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017267.509977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017268.213988:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017268.511974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017269.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017269.513934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017270.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017270.516024:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017271.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017271.397843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.397854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017271.397858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.397891:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.397900:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.397907:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017271.397942:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335317504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017271.397975:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.397979:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017271.397980:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.397985:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.397988:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.397991:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017271.398001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017271.398011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335317632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017271.398021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.398024:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017271.398025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.398029:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.398032:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.398035:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017271.398040:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017271.398047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335317760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017271.398057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.398061:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017271.398062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.398066:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.398068:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.398071:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017271.398075:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017271.398083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335317888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017271.398092:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.398095:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017271.398097:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.398101:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.398103:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.398106:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335317504:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017271.398110:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017271.398117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335318016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017271.398125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.398128:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017271.398130:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.398133:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.398136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.398139:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017271.398143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017271.398151:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335318144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017271.398158:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017271.398161:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017271.398162:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017271.398165:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1535 00000100:00080000:0.0:1734017271.398168:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017271.398171:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017271.398174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017271.398183:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335318272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734017271.398201:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59ca680 x1818247335317504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335317504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335317504:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017271.398285:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335317632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017271.398303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59cb480 x1818247335317632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335317632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335317632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017271.398342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335317760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017271.398357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59cad80 x1818247335317760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335317760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335317760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017271.398421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335317888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017271.398439:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59cb800 x1818247335317888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335317888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398466:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335317888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017271.398477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017271.398490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59cb100 x1818247335318016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335318016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017271.398524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017271.398536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398543:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59c9180 x1818247335318144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398560:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335318144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398568:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017271.398579:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017271.398592:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017271.398597:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017271/real 1734017271] req@ffff8800b59ca300 x1818247335318272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017271.398607:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335318272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017271.398614:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017271.398616:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017271.398619:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318272:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017271.517838:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017272.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017272.519898:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017273.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017273.521888:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017274.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017274.523984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017275.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017275.526021:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017276.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017276.405869:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.405881:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017276.405886:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017276.405899:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.405910:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.405920:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017276.405969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335318400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017276.406008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.406012:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017276.406014:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017276.406019:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.406023:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.406026:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017276.406037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017276.406049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335318528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017276.406059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.406062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017276.406064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017276.406068:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.406071:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.406074:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017276.406079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017276.406090:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335318656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017276.406099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.406102:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017276.406103:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017276.406108:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.406111:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.406114:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017276.406119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017276.406128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335318784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017276.406137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.406140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017276.406142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734017276.406144:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335318400:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017276.406146:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.406148:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.406152:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017276.406156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017276.406165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335318912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017276.406173:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.406175:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017276.406177:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017276.406181:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.406184:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.406188:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017276.406192:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017276.406200:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335319040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017276.406210:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017276.406214:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017276.406215:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017276.406218:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1540 00000100:00080000:0.0:1734017276.406222:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017276.406225:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017276.406229:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017276.406237:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335319168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017276.406243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335318528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017276.406267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335318656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017276.406293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335318784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017276.406309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017276.406327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017276.406342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319168:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017276.527992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017277.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017277.529973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017278.213885:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017278.531953:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017279.213925:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:2.0:1734017279.411969:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734017279.412013:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734017279.412034:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017279.412068:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017279.412077:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017279.412086:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017279.412092:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017279.412101:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017279.412109:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734017279.412188:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59c9180 x1818247335318784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335318784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412255:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335318784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017279.412267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59cb100 x1818247335318912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335318912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335318912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017279.412321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59cb800 x1818247335319040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335319040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412349:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017279.412367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59cad80 x1818247335319168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412384:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335319168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017279.412414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59c9500 x1818247335318400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335318400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335318400:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017279.412465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59c8000 x1818247335318528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335318528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335318528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017279.412513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017276/real 1734017279] req@ffff8800b59ca300 x1818247335318656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017279.412531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335318656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017279.412545:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017279.412549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017279.412554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335318656:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017279.533968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017280.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017280.535947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017281.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017281.413847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.413855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017281.413860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.413869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.413877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.413885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017281.413922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335319296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017281.413954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.413958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017281.413960:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.413965:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.413968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.413973:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017281.413982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017281.413993:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335319424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017281.414003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.414006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017281.414008:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.414011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.414014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.414017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017281.414022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017281.414029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335319552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017281.414037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.414040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017281.414041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.414045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.414047:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.414051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017281.414055:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017281.414055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335319296:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017281.414062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335319680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017281.414070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.414074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017281.414075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.414079:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.414081:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.414084:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017281.414088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017281.414095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335319808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017281.414103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.414106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017281.414107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.414111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.414113:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.414116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017281.414120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734017281.414120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017281.414130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335319936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017281.414138:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017281.414141:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734017281.414141:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59cb480 x1818247335319296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017281.414142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017281.414145:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1545 00000100:00080000:0.0:1734017281.414148:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017281.414151:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017281.414156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017281.414160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335319296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017281.414163:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335320064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734017281.414170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335319296:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017281.414187:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335319424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017281.414203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017281.414209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59ca680 x1818247335319424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017281.414221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335319424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017281.414228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335319424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017281.414240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335319552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017281.414253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017281.414259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59c8e00 x1818247335319552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017281.414269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335319552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017281.414276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335319552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017281.414311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335319680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017281.414326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017281.414332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59caa00 x1818247335319680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017281.414343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335319680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017281.414351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335319680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017281.414361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017281.414373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017281.414378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59c8a80 x1818247335319808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017281.414388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335319808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017281.414395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017281.414406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017281.414417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017281.414423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59c8700 x1818247335319936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017281.414440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335319936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017281.414448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335319936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017281.414458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017281.414469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017281.414475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017281/real 1734017281] req@ffff8800b59c9f80 x1818247335320064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017281.414484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335320064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017281.414491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017281.414493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017281.414496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320064:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017281.538001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017282.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017282.539947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017283.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017283.541961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017284.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017284.543892:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017285.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017285.545950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017286.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017286.421842:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.421852:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017286.421857:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.421871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.421881:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.421891:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017286.421937:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335320192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017286.421975:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.421980:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017286.421981:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.421986:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.421990:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.421993:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017286.422004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017286.422015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335320320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017286.422025:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.422028:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017286.422030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.422034:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.422038:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.422042:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017286.422046:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017286.422057:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335320448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017286.422066:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.422069:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017286.422071:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.422075:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.422077:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.422081:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017286.422085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017286.422093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335320576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017286.422103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.422106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017286.422108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.422112:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.422115:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.422118:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422119:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335320192:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017286.422122:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017286.422131:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335320704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017286.422146:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.422149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017286.422150:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.422154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.422156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.422160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017286.422164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017286.422172:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335320832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017286.422180:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017286.422183:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017286.422184:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017286.422188:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1550 00000100:00080000:0.0:1734017286.422191:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017286.422194:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734017286.422196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017286.422198:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017286.422207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335320960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017286.422223:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59cbb80 x1818247335320192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335320192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335320192:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017286.422272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335320320:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017286.422289:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422295:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59c9c00 x1818247335320320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335320320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422321:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335320320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017286.422328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335320448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017286.422341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422347:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59c9f80 x1818247335320448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335320448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422365:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335320448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017286.422399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335320576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017286.422413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59c8700 x1818247335320576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335320576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335320576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017286.422448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320704:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017286.422461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59c8a80 x1818247335320704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335320704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017286.422495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017286.422510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59caa00 x1818247335320832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335320832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017286.422545:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017286.422558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017286.422563:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017286/real 1734017286] req@ffff8800b59c8e00 x1818247335320960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017286.422573:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335320960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017286.422580:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017286.422583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017286.422585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335320960:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017286.547962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017287.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017287.549975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017288.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017288.551925:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017289.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017289.553940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017290.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017290.555977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017291.214754:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017291.429805:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.429814:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017291.429818:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.429828:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00080000:0.0:1734017291.429837:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.429845:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017291.429881:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335321088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017291.429913:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.429918:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017291.429920:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.429924:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00080000:0.0:1734017291.429927:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.429931:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017291.429940:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017291.429951:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335321216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017291.429960:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.429964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017291.429965:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.429969:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00080000:0.0:1734017291.429972:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.429975:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017291.429979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017291.429987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335321344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017291.429995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.429998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017291.430001:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.430004:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00080000:0.0:1734017291.430007:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.430010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017291.430015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017291.430018:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335321088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017291.430022:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335321472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017291.430031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.430034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017291.430036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.430039:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00080000:0.0:1734017291.430041:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.430045:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017291.430048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017291.430055:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335321600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017291.430064:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.430066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017291.430068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.430071:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00080000:0.0:1734017291.430073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.430077:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017291.430080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734017291.430085:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017291.430087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335321728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017291.430096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017291.430099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017291.430101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017291.430104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1555 00000100:00100000:3.0:1734017291.430106:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59ca680 x1818247335321088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017291.430107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017291.430110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017291.430114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017291.430122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335321856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017291.430125:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335321088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430135:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430138:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430143:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335321088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017291.430150:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335321216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017291.430167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017291.430173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59cb480 x1818247335321216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017291.430184:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335321216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430192:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430198:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335321216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017291.430204:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335321344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017291.430216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017291.430222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59ca300 x1818247335321344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017291.430232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335321344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335321344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017291.430270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335321472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017291.430282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017291.430289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59c8000 x1818247335321472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017291.430299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335321472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335321472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017291.430317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335321600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017291.430328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017291.430334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59c9500 x1818247335321600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017291.430344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335321600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335321600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017291.430363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335321728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017291.430375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017291.430381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59cad80 x1818247335321728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017291.430391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335321728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335321728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017291.430409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335321856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017291.430421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017291.430427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017291/real 1734017291] req@ffff8800b59cb800 x1818247335321856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017291.430437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335321856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017291.430445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017291.430447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017291.430450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335321856:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017291.557957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017292.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017292.559956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017293.213898:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017293.561958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017294.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017294.563946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017295.213928:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017295.565940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017296.213933:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017296.437846:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.437854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017296.437858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.437868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.437877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.437885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017296.437920:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335321984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017296.437954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.437958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017296.437959:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.437964:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.437968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.437971:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017296.437981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017296.437991:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335322112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017296.438001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.438004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017296.438006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.438009:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.438012:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.438015:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017296.438020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017296.438029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335322240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017296.438038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.438040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017296.438042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.438045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.438048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.438051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017296.438055:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017296.438056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335321984:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017296.438063:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335322368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017296.438078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.438082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017296.438083:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.438087:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.438089:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.438092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017296.438096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017296.438103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335322496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017296.438111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.438113:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017296.438115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.438118:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.438121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.438124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734017296.438124:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017296.438128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017296.438136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335322624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017296.438143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017296.438146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734017296.438147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59cb100 x1818247335321984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017296.438148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017296.438151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1560 00000100:00080000:0.0:1734017296.438155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017296.438158:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017296.438162:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017296.438167:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335321984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017296.438169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335322752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734017296.438177:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438181:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335321984:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017296.438192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335322112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017296.438209:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017296.438216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59c9180 x1818247335322112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017296.438227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335322112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017296.438235:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335322112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017296.438247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335322240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017296.438260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017296.438266:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59cb800 x1818247335322240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017296.438278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335322240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017296.438286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335322240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017296.438318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335322368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017296.438330:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017296.438336:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59cad80 x1818247335322368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017296.438346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335322368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017296.438353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335322368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017296.438364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335322496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017296.438376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017296.438381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59c9500 x1818247335322496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017296.438391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335322496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017296.438398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335322496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017296.438411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335322624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017296.438422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017296.438428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59c8000 x1818247335322624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017296.438475:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335322624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017296.438484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335322624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017296.438495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335322752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017296.438511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017296.438518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017296/real 1734017296] req@ffff8800b59ca300 x1818247335322752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017296.438528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335322752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017296.438536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017296.438538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017296.438542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335322752:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017296.567945:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017297.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017297.569935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017298.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017298.571958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017299.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017299.573944:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017300.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017300.575900:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017301.214010:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017301.445793:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.445801:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017301.445806:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.445816:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.445824:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.445832:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017301.445867:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335322880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017301.445900:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.445904:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017301.445907:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.445911:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.445914:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.445918:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017301.445928:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017301.445939:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335323008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017301.445948:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.445952:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017301.445954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.445957:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.445959:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.445963:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017301.445967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017301.445975:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335323136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017301.445984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.445988:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017301.445989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.445993:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.445995:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.445998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017301.446002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017301.446009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335323264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017301.446011:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335322880:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017301.446018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.446021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017301.446023:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.446026:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.446029:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.446032:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017301.446037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017301.446044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335323392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017301.446051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.446054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017301.446055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.446059:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.446061:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.446064:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017301.446068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017301.446075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335323520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734017301.446077:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017301.446083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017301.446086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017301.446087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017301.446091:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1565 00000100:00080000:0.0:1734017301.446094:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017301.446097:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017301.446101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017301.446101:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59cb480 x1818247335322880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017301.446110:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335323648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017301.446120:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335322880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446130:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446134:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446139:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335322880:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017301.446146:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017301.446162:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017301.446169:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59ca680 x1818247335323008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017301.446180:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335323008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446187:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446190:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446193:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017301.446199:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017301.446218:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017301.446225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59c8e00 x1818247335323136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017301.446236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335323136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446246:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017301.446275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017301.446287:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017301.446293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59caa00 x1818247335323264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017301.446303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335323264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017301.446321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335323392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017301.446333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017301.446338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59c8a80 x1818247335323392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017301.446348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335323392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446418:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335323392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017301.446424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335323520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017301.446441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017301.446449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59c8700 x1818247335323520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017301.446459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335323520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335323520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017301.446477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335323648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017301.446489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017301.446495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017301/real 1734017301] req@ffff8800b59c9f80 x1818247335323648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017301.446504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335323648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017301.446511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017301.446513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017301.446516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335323648:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017301.577933:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017302.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017302.579964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017303.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017303.581941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017304.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017304.583946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017305.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017305.585960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017306.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017306.453824:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.453833:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017306.453837:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.453847:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.453855:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.453863:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017306.453898:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335323776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017306.453931:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.453936:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017306.453937:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.453942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.453945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.453948:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017306.453958:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017306.453968:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335323904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017306.453978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.453981:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017306.453983:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.453987:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.453990:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.453993:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017306.453997:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017306.454007:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335324032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017306.454015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.454018:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017306.454020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.454023:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.454025:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.454028:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017306.454032:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017306.454040:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335324160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017306.454042:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335323776:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017306.454049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.454052:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017306.454053:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.454057:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.454060:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.454063:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017306.454067:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017306.454074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335324288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017306.454083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.454086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017306.454087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.454090:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.454093:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.454096:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017306.454100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017306.454107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335324416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734017306.454108:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017306.454115:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017306.454118:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017306.454119:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017306.454122:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1570 00000100:00080000:0.0:1734017306.454125:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017306.454128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454130:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59c9c00 x1818247335323776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734017306.454132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017306.454139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335324544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017306.454149:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335323776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454159:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454162:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454167:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335323776:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017306.454174:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017306.454190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59cbb80 x1818247335323904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017306.454208:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335323904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454218:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335323904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017306.454231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017306.454246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59c9f80 x1818247335324032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017306.454263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335324032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017306.454352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017306.454370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59c8700 x1818247335324160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017306.454389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335324160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017306.454407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335324288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017306.454418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59c8a80 x1818247335324288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017306.454435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335324288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335324288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017306.454453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335324416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017306.454464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59caa00 x1818247335324416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017306.454480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335324416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335324416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017306.454497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335324544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017306.454508:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017306.454514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017306/real 1734017306] req@ffff8800b59c8e00 x1818247335324544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017306.454524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335324544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017306.454531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017306.454534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017306.454537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335324544:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017306.587932:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017307.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017307.589955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017308.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017308.591962:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017309.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017309.593906:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017310.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017310.595962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017311.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017311.461802:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.461810:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017311.461815:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.461824:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.461833:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.461840:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017311.461876:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335324672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017311.461911:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.461915:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017311.461917:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.461922:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.461925:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.461928:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017311.461937:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017311.461948:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335324800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017311.461958:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.461961:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017311.461963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.461967:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.461970:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.461974:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017311.461978:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017311.461986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335324928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017311.461995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.461998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017311.462000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.462003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.462005:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.462009:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017311.462013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017311.462021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335325056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017311.462022:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335324672:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017311.462029:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.462033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017311.462034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.462037:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.462040:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.462043:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017311.462047:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017311.462054:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335325184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017311.462062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.462065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017311.462066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.462070:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.462073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.462077:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017311.462081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017311.462088:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335325312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734017311.462089:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017311.462096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017311.462099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017311.462100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017311.462103:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1575 00000100:00080000:0.0:1734017311.462106:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017311.462109:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462112:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59ca680 x1818247335324672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734017311.462113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017311.462122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335325440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017311.462131:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335324672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462142:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462145:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462150:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335324672:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017311.462157:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017311.462174:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59cb480 x1818247335324800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017311.462231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335324800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017311.462253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017311.462271:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462278:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59ca300 x1818247335324928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017311.462288:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335324928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462298:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335324928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017311.462326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017311.462338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59c8000 x1818247335325056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017311.462355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335325056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017311.462373:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335325184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017311.462384:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59c9500 x1818247335325184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017311.462401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335325184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462410:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335325184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017311.462419:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335325312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017311.462430:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59cad80 x1818247335325312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017311.462445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335325312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335325312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017311.462462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335325440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017311.462474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017311.462480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017311/real 1734017311] req@ffff8800b59cb800 x1818247335325440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017311.462490:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335325440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017311.462504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017311.462506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017311.462509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335325440:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017311.597932:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017312.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017312.599962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017313.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017313.601953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017314.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017314.603832:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017315.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017315.605893:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017316.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017316.469845:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017316.469854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017316.469858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017316.469867:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.469876:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017316.469884:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017316.469920:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335325568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017316.469955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017316.469960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017316.469963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017316.469969:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.469975:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017316.469980:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017316.469992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017316.470011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335325696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017316.470015:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335325568:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017316.470027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017316.470031:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017316.470033:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017316.470038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.470042:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017316.470046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017316.470053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017316.470066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335325824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017316.470078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017316.470082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017316.470085:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734017316.470086:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017316.470091:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.470142:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734017316.470146:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59c9180 x1818247335325568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017316.470148:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017316.470156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017316.470171:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335325568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017316.470172:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335325952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017316.470186:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1734017316.470186:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017316.470191:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1734017316.470192:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734017316.470194:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734017316.470198:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335325568:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017316.470199:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.470203:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017316.470208:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017316.470213:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017316.470223:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335326080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017316.470228:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325696:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734017316.470237:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017316.470242:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017316.470244:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017316.470249:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.470254:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734017316.470256:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017316.470259:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017316.470265:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017316.470268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59cb100 x1818247335325696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017316.470277:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335326208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017316.470286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335325696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017316.470292:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017316.470297:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017316.470299:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017316.470304:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1581 00000100:00080000:0.0:1734017316.470308:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1734017316.470311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017316.470313:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734017316.470315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000008:00100000:0.0:1734017316.470319:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017316.470320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017316.470330:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335326336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017316.470330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017316.470353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017316.470364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59cb800 x1818247335325824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017316.470382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335325824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017316.470394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017316.470398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017316.470403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017316.470412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017316.470436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017316.470445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59cad80 x1818247335325952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017316.470456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335325952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017316.470464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017316.470466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017316.470469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335325952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017316.470488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017316.470501:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017316.470508:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59c9500 x1818247335326080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017316.470518:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335326080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017316.470526:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017316.470528:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017316.470531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017316.470536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017316.470547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017316.470552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59c8000 x1818247335326208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017316.470562:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335326208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017316.470570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017316.470572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017316.470575:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017316.470580:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017316.470592:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017316.470598:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017316/real 1734017316] req@ffff8800b59ca300 x1818247335326336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017316.470609:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335326336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017316.470616:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017316.470618:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017316.470621:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326336:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017316.607844:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017317.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017317.609961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017318.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017318.611957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017319.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017319.613896:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017320.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017320.615945:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017321.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017321.477848:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.477857:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017321.477861:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.477871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1586 00000100:00080000:0.0:1734017321.477880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.477887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017321.477923:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335326464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017321.477957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.477961:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017321.477964:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.477969:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1586 00000100:00080000:0.0:1734017321.477972:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.477975:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017321.477985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017321.477996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335326592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017321.478006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.478043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017321.478044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.478048:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1586 00000100:00080000:0.0:1734017321.478051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.478055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017321.478060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017321.478069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335326720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017321.478079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.478082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017321.478084:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.478087:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1586 00000100:00100000:3.0:1734017321.478087:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335326464:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017321.478090:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.478093:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017321.478097:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017321.478105:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335326848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017321.478114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.478117:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017321.478119:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.478122:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1586 00000100:00080000:0.0:1734017321.478124:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.478128:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017321.478132:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017321.478139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335326976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017321.478147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.478150:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017321.478151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.478155:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1586 00000800:00000100:3.0:1734017321.478156:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017321.478157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.478161:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017321.478165:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017321.478172:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335327104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017321.478178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59cb480 x1818247335326464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017321.478179:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017321.478182:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017321.478184:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017321.478187:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1586 00000100:00080000:0.0:1734017321.478191:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017321.478194:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478197:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335326464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734017321.478198:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017321.478206:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335327232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734017321.478208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478216:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335326464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017321.478223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335326592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017321.478239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59ca680 x1818247335326592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017321.478266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335326592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017321.478274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478279:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335326592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017321.478305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335326720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017321.478319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59c8e00 x1818247335326720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017321.478335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335326720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017321.478343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335326720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017321.478354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335326848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017321.478365:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59caa00 x1818247335326848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017321.478380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335326848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017321.478388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478391:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335326848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017321.478400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017321.478412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59c8a80 x1818247335326976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017321.478427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335326976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017321.478434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478440:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335326976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017321.478444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335327104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017321.478455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59c8700 x1818247335327104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017321.478471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335327104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017321.478479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335327104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017321.478489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335327232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017321.478504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017321.478510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017321/real 1734017321] req@ffff8800b59c9f80 x1818247335327232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017321.478521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335327232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017321.478528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017321.478530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017321.478533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335327232:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017321.617967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017322.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017322.619930:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017323.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017323.621921:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017324.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017324.623941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017325.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017325.626000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017326.213987:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017326.485829:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.485841:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017326.485846:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017326.485859:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.485870:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017326.485880:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017326.485968:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335327360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017326.486005:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.486010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017326.486012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017326.486016:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.486020:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017326.486024:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017326.486036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017326.486050:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335327488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017326.486062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.486065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017326.486067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017326.486071:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.486073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017326.486078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017326.486083:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017326.486094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335327616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017326.486106:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.486109:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017326.486110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017326.486114:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.486117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017326.486121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486123:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335327360:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017326.486125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017326.486135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335327744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017326.486144:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.486147:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017326.486149:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017326.486152:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.486156:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017326.486160:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017326.486165:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017326.486173:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335327872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017326.486180:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.486183:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017326.486185:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017326.486188:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.486192:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734017326.486195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017326.486196:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017326.486200:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017326.486207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335328000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017326.486216:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017326.486219:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017326.486220:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734017326.486220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59cbb80 x1818247335327360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017326.486224:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1591 00000100:00080000:0.0:1734017326.486227:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017326.486231:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017326.486235:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017326.486240:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335327360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017326.486242:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335328128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734017326.486251:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486260:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335327360:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017326.486269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335327488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017326.486290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59c9c00 x1818247335327488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017326.486309:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335327488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017326.486316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335327488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017326.486347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335327616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017326.486361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59c9f80 x1818247335327616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017326.486377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335327616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017326.486385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335327616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017326.486397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335327744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017326.486409:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486415:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59c8700 x1818247335327744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017326.486432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335327744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017326.486440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335327744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017326.486451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335327872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017326.486463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59c8a80 x1818247335327872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017326.486479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335327872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017326.486486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335327872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017326.486497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017326.486509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59caa00 x1818247335328000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017326.486525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335328000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017326.486533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017326.486546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017326.486558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017326.486564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017326/real 1734017326] req@ffff8800b59c8e00 x1818247335328128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017326.486573:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335328128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017326.486580:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017326.486582:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017326.486585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328128:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017326.627995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017327.213996:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017327.629990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017328.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017328.631957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017329.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017329.633950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017330.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017330.635920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017331.213897:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017331.493875:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.493883:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017331.493888:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.493898:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.493906:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.493913:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017331.493949:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335328256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017331.493981:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.493985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017331.493987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.493991:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.493994:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.493997:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017331.494007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017331.494017:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335328384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017331.494027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.494030:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017331.494032:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.494036:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.494039:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.494042:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017331.494047:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017331.494054:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335328512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017331.494063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.494066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017331.494067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.494070:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.494073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.494076:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017331.494080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017331.494088:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335328640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017331.494092:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335328256:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017331.494096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.494100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017331.494101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.494105:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.494107:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.494111:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017331.494115:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017331.494122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335328768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017331.494131:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.494134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017331.494136:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.494139:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.494142:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.494145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017331.494148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017331.494156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335328896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734017331.494160:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017331.494163:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017331.494166:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017331.494168:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017331.494171:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1596 00000100:00080000:0.0:1734017331.494174:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017331.494177:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017331.494181:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017331.494184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59ca680 x1818247335328256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017331.494190:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335329024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017331.494203:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335328256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494218:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494223:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335328256:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017331.494230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335328384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017331.494246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017331.494253:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59cb480 x1818247335328384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017331.494263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335328384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494274:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335328384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017331.494284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335328512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017331.494297:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017331.494303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59ca300 x1818247335328512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017331.494313:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335328512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335328512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017331.494351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335328640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017331.494367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017331.494373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59c8000 x1818247335328640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017331.494384:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335328640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494397:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335328640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017331.494402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017331.494413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017331.494419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59c9500 x1818247335328768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017331.494429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335328768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017331.494447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017331.494458:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017331.494463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59cad80 x1818247335328896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017331.494480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335328896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335328896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017331.494498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017331.494509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017331.494515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017331/real 1734017331] req@ffff8800b59cb800 x1818247335329024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017331.494525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335329024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017331.494532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017331.494534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017331.494537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329024:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017331.637975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017332.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017332.639986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017333.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017333.641987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017334.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017334.643937:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017335.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017335.645991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017336.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017336.501858:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.501867:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017336.501871:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017336.501881:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.501890:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.501898:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017336.501933:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335329152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017336.501966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.501970:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017336.501972:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017336.501976:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.501979:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.501983:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017336.501992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017336.502002:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335329280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017336.502012:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.502015:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017336.502017:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017336.502021:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.502023:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.502027:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017336.502031:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017336.502041:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335329408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017336.502051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.502054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017336.502055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017336.502058:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.502061:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.502065:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017336.502069:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017336.502069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335329152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017336.502076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335329536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017336.502085:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.502088:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017336.502090:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017336.502093:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.502095:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.502099:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017336.502103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017336.502109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335329664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017336.502117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.502120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017336.502122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017336.502125:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.502129:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.502134:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734017336.502135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017336.502138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017336.502145:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335329792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017336.502153:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017336.502156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017336.502157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734017336.502158:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59cb100 x1818247335329152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017336.502161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1601 00000100:00080000:0.0:1734017336.502164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017336.502167:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017336.502171:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734017336.502177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335329152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017336.502178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335329920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734017336.502188:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502196:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335329152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734017336.502203:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335329280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017336.502219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017336.502226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59c9180 x1818247335329280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017336.502237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335329280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017336.502244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335329280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017336.502256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335329408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017336.502269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017336.502275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59cb800 x1818247335329408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017336.502286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335329408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017336.502293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502296:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335329408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017336.502325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335329536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017336.502338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017336.502344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59cad80 x1818247335329536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017336.502354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335329536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017336.502361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335329536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017336.502371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734017336.502382:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017336.502388:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59c9500 x1818247335329664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017336.502398:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335329664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017336.502405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734017336.502415:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017336.502431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017336.502437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59c8000 x1818247335329792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017336.502447:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335329792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017336.502455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734017336.502464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734017336.502476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734017336.502481:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017336/real 1734017336] req@ffff8800b59ca300 x1818247335329920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734017336.502491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335329920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734017336.502498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734017336.502500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734017336.502503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335329920:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017336.647941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017337.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017337.649997:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017338.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017338.651965:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017339.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017339.653926:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017340.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017340.655942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017341.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017341.509810:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.509820:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017341.509825:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.509838:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.509849:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017341.509859:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017341.510100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335330048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017341.510147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.510154:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017341.510158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.510165:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.510177:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017341.510187:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017341.510199:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017341.510218:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335330176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017341.510220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335330048:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017341.510234:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.510238:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017341.510241:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.510246:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.510250:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017341.510255:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017341.510261:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017341.510272:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335330304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017341.510284:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.510289:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017341.510291:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.510297:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.510301:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017341.510306:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017341.510313:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017341.510325:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335330432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017341.510330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335330176:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734017341.510339:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.510344:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017341.510346:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.510351:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.510355:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017341.510360:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017341.510362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335330304:192.168.201.104@tcp:8:kworker.0 00000008:00100000:0.0:1734017341.510367:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017341.510378:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335330560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017341.510389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335330432:192.168.201.104@tcp:8:kworker.0 00000100:00080000:0.0:1734017341.510393:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.510397:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017341.510400:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.510405:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.510410:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734017341.510414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335330560:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734017341.510415:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017341.510422:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017341.510434:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335330688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017341.510453:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017341.510458:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017341.510460:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017341.510466:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1606 00000100:00080000:0.0:1734017341.510470:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017341.510475:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734017341.510477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335330688:192.168.201.104@tcp:8:kworker.0 00000008:00100000:0.0:1734017341.510482:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017341.510495:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335330816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734017341.510536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335330816:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017341.657877:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017342.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017342.659945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017343.213776:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017343.661973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017344.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:2.0:1734017344.516041:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734017344.516115:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734017344.516143:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017344.516193:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017344.516208:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017344.516219:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017344.516220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59ca680 x1818247335330176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000800:00000100:2.0:1734017344.516231:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017344.516244:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017344.516252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335330176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000800:00000100:2.0:1734017344.516261:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00080000:1.0:1734017344.516265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335330176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017344.516283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59c8e00 x1818247335330304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017344.516297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335330304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017344.516305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335330304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017344.516319:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59cb480 x1818247335330048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017344.516330:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335330048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017344.516337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516340:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335330048:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017344.516381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59c9f80 x1818247335330816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017344.516412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335330816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017344.516435:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335330816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017344.516455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59c8700 x1818247335330688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017344.516474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335330688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017344.516486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335330688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017344.516506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59c8a80 x1818247335330560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017344.516524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335330560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017344.516536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516546:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335330560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017344.516556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017341/real 1734017344] req@ffff8800b59caa00 x1818247335330432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017344.516575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335330432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017344.516597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017344.516600:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017344.516605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335330432:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017344.663929:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017345.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017345.665973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017346.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017346.517847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.517855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017346.517859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.517869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.517878:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017346.517886:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017346.517921:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335330944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017346.517954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.517958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017346.517961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.517966:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.517969:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017346.517972:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017346.517983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017346.517993:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335331072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017346.518002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.518006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017346.518008:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.518011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.518014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017346.518017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017346.518022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017346.518031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335331200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017346.518040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.518043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017346.518044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.518047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.518050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017346.518053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017346.518058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017346.518064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335330944:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017346.518066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335331328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017346.518076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.518079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017346.518081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.518084:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.518087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017346.518090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017346.518094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017346.518101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335331456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017346.518109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.518112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017346.518113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.518116:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.518119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017346.518122:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017346.518126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017346.518132:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017346.518133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335331584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017346.518141:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017346.518144:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017346.518145:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017346.518148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1611 00000100:00080000:0.0:1734017346.518152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017346.518152:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59c9c00 x1818247335330944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017346.518155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017346.518159:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017346.518167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335331712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017346.518171:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335330944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518181:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518185:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518189:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335330944:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017346.518197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017346.518213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017346.518220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59cbb80 x1818247335331072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017346.518231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335331072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518244:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017346.518252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017346.518265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017346.518272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59caa00 x1818247335331200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017346.518282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335331200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518290:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017346.518317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017346.518329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017346.518335:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59c8a80 x1818247335331328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017346.518344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335331328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017346.518363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335331456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017346.518375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017346.518381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59c8700 x1818247335331456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017346.518392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335331456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335331456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017346.518410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335331584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017346.518421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017346.518427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59c9f80 x1818247335331584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017346.518436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335331584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518449:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335331584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017346.518454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335331712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017346.518466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017346.518472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017346/real 1734017346] req@ffff8800b59cb480 x1818247335331712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017346.518481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335331712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017346.518488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017346.518491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017346.518494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335331712:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017346.667909:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017347.213757:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000008:00020000:2.0:1734017347.471750:0:4737:0:(osc_cache.c:908:osc_extent_wait()) extent ffff880136be3540@{[0 -> 0/1023], [3|0|+|rpc|wiuY|ffff8800a85d81a0], [28672|1|+|-|ffff8800a87a3180|1024|ffff880136a28000]} lustre-OST0000-osc-ffff8800b588c800: wait ext to 0 timedout, recovery in progress? 00000008:00020000:2.0:1734017347.480807:0:4737:0:(osc_cache.c:908:osc_extent_wait()) ### extent: ffff880136be3540 ns: lustre-OST0000-osc-ffff8800b588c800 lock: ffff8800a87a3180/0xd17286ca2ee49827 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800029400020000 nid: local remote: 0x979ba2bc5598ad83 expref: -99 pid: 9551 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1734017347.669957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017348.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017348.671947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017349.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017349.673964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017350.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017350.675993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017351.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017351.525807:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.525817:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017351.525822:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017351.525835:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1616 00000100:00080000:0.0:1734017351.525846:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.525856:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017351.525905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335331840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017351.525944:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.525948:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017351.525950:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017351.525955:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1616 00000100:00080000:0.0:1734017351.525958:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.525962:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017351.525974:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017351.525986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335331968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017351.525996:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.526000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017351.526002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017351.526006:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1616 00000100:00080000:0.0:1734017351.526008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.526012:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017351.526016:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017351.526025:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335332096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017351.526034:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.526037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017351.526038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017351.526042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1616 00000100:00080000:0.0:1734017351.526045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.526048:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017351.526052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017351.526060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335332224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017351.526070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.526073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017351.526075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017351.526076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335331840:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017351.526080:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1616 00000100:00080000:0.0:1734017351.526082:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.526086:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017351.526090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017351.526098:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335332352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017351.526106:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.526109:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017351.526111:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017351.526114:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1616 00000100:00080000:0.0:1734017351.526117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.526120:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017351.526125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017351.526133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335332480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017351.526140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017351.526143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017351.526145:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017351.526148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1616 00000800:00000100:1.0:1734017351.526151:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017351.526152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017351.526156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017351.526160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017351.526172:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335332608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017351.526175:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59c8e00 x1818247335331840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526196:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335331840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526217:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335331840:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017351.526225:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017351.526243:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017351.526249:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59ca680 x1818247335331968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335331968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526268:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335331968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017351.526281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017351.526300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017351.526307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59ca300 x1818247335332096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335332096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017351.526358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017351.526370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017351.526376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59c8000 x1818247335332224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335332224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017351.526405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335332352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017351.526417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017351.526422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59c9500 x1818247335332352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335332352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335332352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017351.526458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335332480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017351.526470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017351.526476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59cad80 x1818247335332480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335332480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335332480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017351.526548:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335332608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017351.526564:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017351.526571:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017351/real 1734017351] req@ffff8800b59cb800 x1818247335332608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017351.526581:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335332608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017351.526589:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017351.526593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017351.526596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335332608:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017351.677988:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017352.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017352.679952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017353.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017353.681922:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017354.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017354.684004:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017355.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017355.685962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017356.213942:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017356.533825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.533834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017356.533839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017356.533848:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.533858:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017356.533865:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017356.533901:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335332736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017356.533933:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.533938:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017356.533940:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017356.533945:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.533948:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017356.533952:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017356.533961:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017356.533972:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335332864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017356.533982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.533985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017356.533987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017356.533992:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.533994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017356.533998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017356.534002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017356.534011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335332992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017356.534019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.534022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017356.534024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017356.534027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.534029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017356.534033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017356.534037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017356.534045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335333120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017356.534053:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.534056:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017356.534058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017356.534058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335332736:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017356.534061:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.534064:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017356.534067:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017356.534072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017356.534078:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335333248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017356.534088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.534091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017356.534093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017356.534096:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.534111:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017356.534114:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017356.534118:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017356.534124:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017356.534126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335333376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017356.534135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017356.534138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017356.534140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017356.534143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1621 00000100:00080000:0.0:1734017356.534146:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017356.534146:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59c9180 x1818247335332736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017356.534150:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017356.534153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017356.534161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335333504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017356.534165:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335332736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534176:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534179:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534184:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335332736:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017356.534192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017356.534209:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017356.534216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59cb100 x1818247335332864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017356.534227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335332864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534235:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017356.534247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017356.534263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017356.534270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59cb800 x1818247335332992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017356.534280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335332992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335332992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017356.534319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335333120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017356.534332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017356.534337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59cad80 x1818247335333120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017356.534347:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335333120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534355:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335333120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017356.534365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335333248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017356.534377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017356.534382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59c9500 x1818247335333248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017356.534399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335333248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534407:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534409:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335333248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017356.534458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335333376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017356.534473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017356.534480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59c8000 x1818247335333376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017356.534491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335333376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335333376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017356.534509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335333504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017356.534521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017356.534527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017356/real 1734017356] req@ffff8800b59ca300 x1818247335333504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017356.534537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335333504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017356.534544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017356.534546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017356.534549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335333504:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017356.687910:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017357.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017357.689946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017358.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017358.692008:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017359.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017359.693970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017360.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017360.695962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017361.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017361.541829:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.541843:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017361.541848:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.541857:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00080000:0.0:1734017361.541866:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.541875:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017361.541910:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335333632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017361.541944:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.541948:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017361.541950:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.541955:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00080000:0.0:1734017361.541958:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.541961:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017361.541971:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017361.541981:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335333760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017361.541991:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.541995:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017361.541996:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.542000:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00080000:0.0:1734017361.542003:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.542006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017361.542010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017361.542018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335333888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017361.542026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.542029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017361.542031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.542035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00080000:0.0:1734017361.542037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.542041:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017361.542045:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017361.542047:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335333632:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017361.542053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335334016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017361.542061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.542064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017361.542066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.542069:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00080000:0.0:1734017361.542072:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.542075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017361.542079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017361.542086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335334144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017361.542093:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.542096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017361.542098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.542101:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00080000:0.0:1734017361.542104:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.542108:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017361.542111:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017361.542115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017361.542119:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335334272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017361.542127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017361.542131:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017361.542132:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017361.542135:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1626 00000100:00100000:1.0:1734017361.542135:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59ca680 x1818247335333632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017361.542138:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017361.542141:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017361.542145:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017361.542154:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335334400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017361.542155:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335333632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542166:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542169:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542174:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335333632:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017361.542181:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335333760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017361.542198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017361.542204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59c8e00 x1818247335333760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017361.542216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335333760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542229:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335333760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017361.542235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335333888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017361.542252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017361.542258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59cb480 x1818247335333888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017361.542269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335333888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335333888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017361.542307:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017361.542319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017361.542325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59c9f80 x1818247335334016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017361.542367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335334016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017361.542386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335334144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017361.542401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017361.542408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59c8700 x1818247335334144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017361.542426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335334144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542433:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335334144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017361.542443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335334272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017361.542455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017361.542461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59c8a80 x1818247335334272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017361.542471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335334272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335334272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017361.542488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335334400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017361.542500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017361.542506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017361/real 1734017361] req@ffff8800b59caa00 x1818247335334400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017361.542516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335334400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017361.542523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017361.542526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017361.542529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335334400:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017361.697942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017362.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017362.699934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017363.213758:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017363.701942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017364.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017364.703927:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017365.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017365.705943:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017366.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017366.549838:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017366.549847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017366.549852:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.549861:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.549869:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017366.549877:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017366.549914:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335334528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017366.549948:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017366.549952:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017366.549954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.549958:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.549961:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017366.549965:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017366.549975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017366.549986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335334656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017366.549996:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017366.549999:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017366.550001:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.550004:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.550006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017366.550010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017366.550014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017366.550024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335334784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017366.550033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017366.550036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017366.550037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.550041:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.550044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017366.550046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335334528:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017366.550047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017366.550051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017366.550058:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335334912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017366.550067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017366.550070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017366.550072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.550076:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.550078:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017366.550082:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017366.550086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017366.550093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335335040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017366.550101:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017366.550104:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017366.550105:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.550108:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.550111:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734017366.550113:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017366.550114:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017366.550118:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017366.550125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335335168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017366.550133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017366.550134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59cbb80 x1818247335334528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017366.550135:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017366.550137:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017366.550140:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1631 00000100:00080000:0.0:1734017366.550143:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017366.550146:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017366.550150:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017366.550154:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335334528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017366.550158:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335335296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017366.550164:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335334528:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017366.550179:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017366.550195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017366.550201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59c9c00 x1818247335334656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017366.550213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335334656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017366.550220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017366.550295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017366.550317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017366.550325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59caa00 x1818247335334784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017366.550337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335334784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017366.550345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550347:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017366.550355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017366.550367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017366.550372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59c8a80 x1818247335334912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017366.550382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335334912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017366.550390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550395:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335334912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017366.550401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017366.550413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017366.550418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59c8700 x1818247335335040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017366.550435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335335040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017366.550443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017366.550453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017366.550464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017366.550470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59c9f80 x1818247335335168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017366.550480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335335168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017366.550487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017366.550497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017366.550510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017366.550516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017366/real 1734017366] req@ffff8800b59cb480 x1818247335335296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017366.550526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335335296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017366.550533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017366.550536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017366.550539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335296:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017366.707936:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017367.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017367.709943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017368.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017368.711881:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017369.213948:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017369.713914:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017370.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017370.715919:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017371.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017371.557822:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.557834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017371.557839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.557853:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.557863:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017371.557872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017371.557919:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335335424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017371.557958:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.557963:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017371.557964:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.557969:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.557973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017371.557977:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017371.557987:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017371.557999:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335335552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017371.558008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.558012:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017371.558014:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.558018:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.558022:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017371.558026:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017371.558031:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017371.558040:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335335680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017371.558048:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.558052:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017371.558053:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.558057:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.558059:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017371.558063:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017371.558067:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017371.558075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335335808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017371.558084:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.558087:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017371.558089:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.558093:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.558095:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017371.558098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335335424:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017371.558099:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017371.558103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017371.558111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335335936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017371.558125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.558129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017371.558130:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.558134:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.558136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017371.558140:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017371.558145:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017371.558153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335336064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017371.558197:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017371.558200:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017371.558202:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017371.558206:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1636 00000100:00080000:0.0:1734017371.558209:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734017371.558211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017371.558213:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017371.558217:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017371.558229:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335336192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017371.558235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59c8e00 x1818247335335424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335335424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558266:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335335424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017371.558284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335335552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017371.558303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017371.558310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59ca680 x1818247335335552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335335552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335335552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017371.558341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335335680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017371.558357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017371.558364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59ca300 x1818247335335680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335335680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335335680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017371.558414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335335808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017371.558427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017371.558433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59c8000 x1818247335335808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335335808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335335808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017371.558463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017371.558476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017371.558482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59c9500 x1818247335335936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335335936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335335936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017371.558518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017371.558529:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017371.558535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59cad80 x1818247335336064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335336064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017371.558563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017371.558576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017371.558582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017371/real 1734017371] req@ffff8800b59cb800 x1818247335336192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017371.558591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335336192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017371.558599:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017371.558602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017371.558605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336192:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017371.717940:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017372.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017372.719979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017373.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017373.721949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017374.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017374.723895:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017375.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017375.725958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017376.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017376.565811:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017376.565820:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017376.565825:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.565835:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00080000:0.0:1734017376.565845:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.565852:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017376.565888:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335336320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017376.565922:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017376.565926:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017376.565927:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.565932:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00080000:0.0:1734017376.565935:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.565938:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017376.565947:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017376.565958:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335336448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017376.565967:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017376.565970:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017376.565972:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.565976:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00080000:0.0:1734017376.565978:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.565982:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017376.565986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017376.565996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335336576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017376.566006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017376.566009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017376.566010:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.566014:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00080000:0.0:1734017376.566016:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.566019:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017376.566023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017376.566031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335336704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017376.566040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017376.566042:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335336320:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017376.566043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017376.566044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.566048:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00080000:0.0:1734017376.566050:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.566053:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017376.566057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017376.566064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335336832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017376.566100:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017376.566103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017376.566104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.566108:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00080000:0.0:1734017376.566121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.566124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017376.566129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017376.566134:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017376.566139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335336960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017376.566147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017376.566151:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017376.566152:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017376.566156:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1641 00000100:00100000:1.0:1734017376.566157:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59cb100 x1818247335336320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017376.566158:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017376.566161:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017376.566165:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017376.566173:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335337088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017376.566176:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335336320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566187:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566190:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566195:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335336320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017376.566203:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335336448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017376.566219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017376.566226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59c9180 x1818247335336448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017376.566237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335336448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335336448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017376.566256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335336576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017376.566273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017376.566279:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59cb800 x1818247335336576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017376.566290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335336576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335336576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017376.566328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335336704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017376.566340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017376.566346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59cad80 x1818247335336704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017376.566356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335336704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335336704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017376.566374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017376.566385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017376.566391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59c9500 x1818247335336832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017376.566408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335336832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017376.566427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017376.566438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017376.566444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59c8000 x1818247335336960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017376.566453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335336960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566461:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335336960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017376.566471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017376.566482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017376.566488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017376/real 1734017376] req@ffff8800b59ca300 x1818247335337088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017376.566498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335337088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017376.566505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017376.566508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017376.566511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337088:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017376.727932:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017377.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017377.729952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017378.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017378.731907:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017379.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017379.733996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017380.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017380.735979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017381.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017381.573823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.573831:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017381.573835:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017381.573846:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:0.0:1734017381.573854:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017381.573862:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017381.573898:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335337216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017381.573931:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.573935:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017381.573937:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017381.573942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:0.0:1734017381.573945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017381.573949:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017381.573959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017381.573969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335337344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017381.573979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.573983:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017381.573984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017381.574022:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:0.0:1734017381.574025:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017381.574029:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017381.574034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017381.574043:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335337472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017381.574054:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.574057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017381.574059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017381.574059:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335337216:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017381.574063:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:0.0:1734017381.574066:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017381.574069:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017381.574073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017381.574081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335337600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017381.574089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.574093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017381.574094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017381.574098:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:0.0:1734017381.574112:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017381.574116:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017381.574120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1734017381.574125:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017381.574128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335337728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017381.574137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.574140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017381.574141:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017381.574144:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:0.0:1734017381.574147:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017381.574147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59ca680 x1818247335337216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017381.574151:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017381.574154:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017381.574161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335337856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017381.574166:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335337216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017381.574170:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017381.574174:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017381.574175:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734017381.574176:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017381.574178:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1646 00000100:00080000:1.0:1734017381.574179:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734017381.574181:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017381.574184:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574184:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335337216:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017381.574188:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017381.574191:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335337344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017381.574197:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335337984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017381.574207:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574214:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59c8e00 x1818247335337344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017381.574225:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335337344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017381.574233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017381.574236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017381.574239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335337344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017381.574264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335337472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017381.574280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574286:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59cb480 x1818247335337472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017381.574297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335337472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017381.574305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017381.574307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017381.574310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335337472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017381.574316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335337600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017381.574327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59c9f80 x1818247335337600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017381.574343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335337600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017381.574350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017381.574352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017381.574355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335337600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017381.574360:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017381.574373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59c8700 x1818247335337728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017381.574396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335337728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017381.574403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017381.574405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017381.574408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017381.574414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017381.574425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59c8a80 x1818247335337856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017381.574441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335337856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017381.574448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017381.574450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017381.574453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017381.574459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017381.574471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017381.574477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017381/real 1734017381] req@ffff8800b59caa00 x1818247335337984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017381.574486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335337984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017381.574493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017381.574496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017381.574499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335337984:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017381.737955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017382.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017382.739956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017383.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017383.742016:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017384.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017384.744022:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017385.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017385.746007:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017386.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017386.581834:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.581843:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017386.581847:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017386.581857:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.581866:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.581874:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017386.581938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335338112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017386.581970:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.581974:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017386.581977:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017386.581981:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.581985:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.581988:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017386.581998:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017386.582009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335338240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017386.582019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.582023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017386.582024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017386.582028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.582030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.582033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017386.582038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017386.582047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335338368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017386.582056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.582059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017386.582060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017386.582063:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.582066:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.582070:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017386.582074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017386.582082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335338496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017386.582087:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335338112:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017386.582092:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.582095:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017386.582097:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017386.582100:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.582103:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.582106:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017386.582110:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017386.582117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335338624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017386.582125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.582127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017386.582129:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017386.582132:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.582144:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.582148:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017386.582152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017386.582154:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017386.582160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335338752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017386.582168:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017386.582171:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017386.582173:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017386.582175:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59c9c00 x1818247335338112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017386.582177:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1651 00000100:00080000:0.0:1734017386.582180:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017386.582183:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017386.582188:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017386.582195:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335338880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017386.582195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335338112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582205:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582208:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582213:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335338112:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017386.582220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335338240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017386.582237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017386.582244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59cbb80 x1818247335338240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017386.582255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335338240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582266:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335338240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017386.582275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335338368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017386.582291:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017386.582297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59caa00 x1818247335338368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017386.582308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335338368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582321:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335338368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017386.582345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335338496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017386.582357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017386.582363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59c8a80 x1818247335338496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017386.582373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335338496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335338496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017386.582391:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335338624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017386.582403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017386.582409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59c8700 x1818247335338624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017386.582426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335338624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582433:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335338624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017386.582444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335338752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017386.582455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017386.582461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59c9f80 x1818247335338752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017386.582471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335338752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335338752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017386.582488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335338880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017386.582500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017386.582506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017386/real 1734017386] req@ffff8800b59cb480 x1818247335338880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017386.582516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335338880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017386.582523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017386.582526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017386.582529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335338880:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017386.747984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017387.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017387.749991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017388.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017388.751981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017389.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017389.753967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017390.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017390.756004:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017391.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017391.589797:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.589805:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017391.589810:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017391.589844:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.589853:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.589861:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017391.589896:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335339008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017391.589928:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.589933:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017391.589935:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017391.589939:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.589942:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.589946:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017391.589955:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017391.589966:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335339136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017391.589976:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.589979:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017391.589981:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017391.589985:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.589987:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.589991:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017391.589995:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017391.590003:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335339264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017391.590011:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.590014:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017391.590016:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017391.590019:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.590021:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.590025:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017391.590028:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017391.590036:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335339392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017391.590045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.590048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017391.590049:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017391.590050:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335339008:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017391.590054:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.590056:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.590059:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017391.590063:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017391.590070:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335339520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017391.590078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.590081:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017391.590082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017391.590086:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.590089:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.590092:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017391.590096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017391.590103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335339648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017391.590111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017391.590113:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017391.590115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017391.590116:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017391.590118:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1656 00000100:00080000:0.0:1734017391.590121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017391.590124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017391.590128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017391.590137:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335339776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017391.590137:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59c8e00 x1818247335339008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590156:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335339008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590167:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590170:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590175:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335339008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017391.590182:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335339136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017391.590199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017391.590205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59ca680 x1818247335339136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590217:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335339136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590230:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335339136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017391.590236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335339264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017391.590257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017391.590263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59ca300 x1818247335339264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335339264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335339264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017391.590315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335339392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017391.590327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017391.590333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59c8000 x1818247335339392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335339392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335339392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017391.590361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335339520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017391.590372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017391.590378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59c9500 x1818247335339520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335339520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335339520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017391.590413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335339648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017391.590424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017391.590430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59cad80 x1818247335339648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335339648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335339648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017391.590457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335339776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017391.590468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017391.590474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017391/real 1734017391] req@ffff8800b59cb800 x1818247335339776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017391.590484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335339776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017391.590491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017391.590494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017391.590497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335339776:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017391.758023:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017392.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017392.760016:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017393.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017393.762008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017394.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017394.763961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017395.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017395.765987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017396.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017396.597823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017396.597831:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017396.597836:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.597845:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00080000:0.0:1734017396.597854:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.597861:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017396.597897:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335339904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017396.597931:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017396.597935:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017396.597937:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.597941:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00080000:0.0:1734017396.597944:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.597948:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017396.597958:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017396.597969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335340032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017396.597979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017396.597982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017396.597985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.597988:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00080000:0.0:1734017396.597991:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.597994:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017396.597999:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017396.598008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335340160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017396.598017:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017396.598018:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335339904:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017396.598020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017396.598022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.598025:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00080000:0.0:1734017396.598028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.598031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017396.598035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017396.598043:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335340288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017396.598051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017396.598054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017396.598055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.598059:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00080000:0.0:1734017396.598061:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.598065:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017396.598069:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017396.598076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335340416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017396.598085:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017396.598088:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017396.598090:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.598093:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00080000:0.0:1734017396.598096:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.598099:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017396.598099:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017396.598103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017396.598111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335340544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017396.598118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017396.598121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017396.598123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017396.598126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1661 00000100:00100000:1.0:1734017396.598126:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59c9180 x1818247335339904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017396.598128:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017396.598131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017396.598135:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017396.598143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335340672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017396.598147:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335339904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598158:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598161:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598166:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335339904:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017396.598174:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017396.598196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017396.598203:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59cb100 x1818247335340032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017396.598215:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335340032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598225:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598229:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017396.598252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017396.598270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017396.598276:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59cb800 x1818247335340160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017396.598287:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335340160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598295:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017396.598306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017396.598317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017396.598323:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59cad80 x1818247335340288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017396.598333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335340288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017396.598351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335340416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017396.598364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017396.598370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59c9500 x1818247335340416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017396.598387:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335340416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335340416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017396.598405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335340544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017396.598416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017396.598422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59c8000 x1818247335340544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017396.598432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335340544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335340544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017396.598450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335340672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017396.598462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017396.598468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017396/real 1734017396] req@ffff8800b59ca300 x1818247335340672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017396.598478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335340672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017396.598486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017396.598490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017396.598495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335340672:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017396.767982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017397.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017397.770005:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017398.213769:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017398.771976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017399.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017399.773961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017400.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017400.775982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017401.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017401.605818:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.605827:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017401.605831:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.605840:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.605849:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017401.605857:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017401.605893:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335340800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017401.605925:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.605929:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017401.605931:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.605936:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.605939:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017401.605942:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017401.605952:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017401.605962:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335340928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017401.605972:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.605975:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017401.605977:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.605980:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.605983:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017401.605986:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017401.605990:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017401.605998:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335341056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017401.606007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.606010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017401.606012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.606015:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.606017:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017401.606021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017401.606025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017401.606028:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335340800:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017401.606032:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335341184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017401.606041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.606044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017401.606046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.606049:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.606051:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017401.606055:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017401.606058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017401.606066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335341312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017401.606073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.606076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017401.606078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.606081:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.606084:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017401.606088:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017401.606092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017401.606096:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017401.606099:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335341440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017401.606107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017401.606110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017401.606112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017401.606115:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1666 00000100:00080000:0.0:1734017401.606117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017401.606117:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59ca680 x1818247335340800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017401.606121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017401.606125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017401.606133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335341568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017401.606137:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335340800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606147:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606150:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606155:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335340800:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017401.606162:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017401.606179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017401.606186:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59c8e00 x1818247335340928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017401.606197:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335340928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606205:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606207:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606210:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335340928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017401.606216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017401.606232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017401.606238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59cb480 x1818247335341056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017401.606249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335341056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017401.606287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017401.606300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017401.606306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59c9f80 x1818247335341184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017401.606315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335341184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017401.606334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335341312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017401.606346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017401.606351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59c8700 x1818247335341312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017401.606368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335341312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606381:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335341312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017401.606386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335341440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017401.606397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017401.606403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59c8a80 x1818247335341440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017401.606413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335341440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606420:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335341440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017401.606430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335341568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017401.606442:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017401.606447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017401/real 1734017401] req@ffff8800b59caa00 x1818247335341568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017401.606457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335341568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017401.606464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017401.606467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017401.606470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335341568:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017401.777969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017402.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017402.779938:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017403.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017403.781989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017404.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017404.783973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017405.213903:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017405.785978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017406.213943:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017406.613836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.613846:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017406.613851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.613864:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.613875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017406.613885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017406.613931:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335341696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017406.613971:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.613976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017406.613978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.613982:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.613986:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017406.613990:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017406.614002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017406.614013:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335341824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017406.614022:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.614026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017406.614028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.614032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.614035:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017406.614038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017406.614043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017406.614055:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335341952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017406.614064:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.614067:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017406.614068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.614072:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.614075:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017406.614079:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017406.614083:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017406.614093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335342080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017406.614099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335341696:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017406.614103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.614106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017406.614108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.614112:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.614114:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017406.614118:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017406.614122:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017406.614131:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335342208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017406.614138:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.614141:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017406.614143:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.614147:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.614160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017406.614163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017406.614168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017406.614176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335342336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017406.614184:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017406.614188:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017406.614190:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017406.614194:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1671 00000100:00080000:0.0:1734017406.614197:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017406.614200:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341824:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734017406.614201:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017406.614205:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017406.614215:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335342464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017406.614226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017406.614252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017406.614268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335342208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017406.614283:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335342336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017406.614299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335342464:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017406.787967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017407.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017407.790050:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017408.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017408.791999:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017409.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:2.0:1734017409.619986:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:2.0:1734017409.620042:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1734017409.620065:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017409.620115:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017409.620129:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017409.620140:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017409.620151:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017409.620163:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:2.0:1734017409.620175:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734017409.620197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59c8a80 x1818247335342080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335342080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017409.620282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59c8700 x1818247335342208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335342208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335342208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017409.620337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59c9f80 x1818247335342336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335342336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335342336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017409.620389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59cb480 x1818247335342464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335342464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620420:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335342464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017409.620440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59cbb80 x1818247335341696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335341696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335341696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017409.620495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59c9c00 x1818247335341824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335341824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620596:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620601:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017409.620611:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017406/real 1734017409] req@ffff8800b59caa00 x1818247335341952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017409.620627:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335341952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017409.620640:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017409.620645:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017409.620650:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335341952:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017409.793928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017410.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017410.795959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017411.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017411.621841:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.621849:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017411.621854:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.621864:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.621873:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.621880:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017411.621916:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335342592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017411.621948:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.621953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017411.621954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.621959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.621963:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.621966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017411.621975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017411.621986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335342720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017411.621995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.621998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017411.622000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.622003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.622006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.622010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017411.622014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017411.622021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335342848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017411.622030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.622032:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017411.622034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.622038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.622041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.622044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017411.622048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017411.622055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335342592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017411.622056:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335342976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017411.622065:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.622068:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017411.622069:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.622073:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.622075:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.622079:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017411.622082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017411.622089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335343104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017411.622097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.622100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017411.622101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.622105:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.622107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.622111:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017411.622115:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017411.622122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335343232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:2.0:1734017411.622127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017411.622131:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017411.622134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017411.622135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017411.622139:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1676 00000100:00080000:0.0:1734017411.622141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017411.622144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59caa00 x1818247335342592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734017411.622148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017411.622155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335343360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017411.622166:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335342592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622176:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622179:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622184:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335342592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017411.622191:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017411.622208:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59c9c00 x1818247335342720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017411.622226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335342720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017411.622245:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017411.622258:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622264:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59cbb80 x1818247335342848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017411.622274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335342848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622285:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017411.622315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017411.622328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59cb480 x1818247335342976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017411.622343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335342976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335342976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017411.622361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335343104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017411.622372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59c9f80 x1818247335343104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017411.622388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335343104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335343104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017411.622406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335343232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017411.622417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59c8700 x1818247335343232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017411.622433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335343232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335343232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017411.622449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335343360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017411.622461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017411.622466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017411/real 1734017411] req@ffff8800b59c8a80 x1818247335343360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017411.622505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335343360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017411.622513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017411.622515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017411.622518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335343360:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017411.797928:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017412.213912:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017412.799989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017413.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017413.801967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017414.213800:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017414.803982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017415.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017415.805975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017416.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017416.629826:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.629834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017416.629838:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.629848:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.629856:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.629864:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017416.629899:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335343488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017416.629931:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.629935:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017416.629937:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.629941:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.629944:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.629948:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017416.629959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017416.629969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335343616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017416.629979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.629982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017416.629984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.629988:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.629990:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.629993:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017416.629998:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017416.630006:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335343744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017416.630014:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.630017:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017416.630018:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.630022:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.630024:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.630028:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017416.630031:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017416.630031:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335343488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017416.630039:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335343872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017416.630048:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.630051:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017416.630054:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.630057:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.630059:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.630062:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017416.630066:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017416.630075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335344000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017416.630083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.630086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017416.630088:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.630091:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.630093:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.630097:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:2.0:1734017416.630097:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017416.630101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017416.630108:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335344128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017416.630116:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017416.630119:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1734017416.630119:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59c8e00 x1818247335343488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017416.630121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017416.630124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1681 00000100:00080000:0.0:1734017416.630127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017416.630130:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017416.630134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017416.630138:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335343488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017416.630142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335344256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017416.630148:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630152:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630157:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335343488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017416.630164:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335343616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017416.630180:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017416.630187:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59ca680 x1818247335343616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017416.630198:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335343616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017416.630206:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630208:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335343616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017416.630217:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335343744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017416.630230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017416.630236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59ca300 x1818247335343744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017416.630254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335343744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017416.630262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335343744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017416.630293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335343872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017416.630305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017416.630311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59c8000 x1818247335343872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017416.630320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335343872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017416.630328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335343872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017416.630339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017416.630351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017416.630356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59c9500 x1818247335344000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017416.630366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335344000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017416.630374:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017416.630386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017416.630434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017416.630441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59cad80 x1818247335344128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017416.630452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335344128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017416.630460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017416.630470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017416.630482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017416.630488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017416/real 1734017416] req@ffff8800b59cb800 x1818247335344256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017416.630498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335344256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017416.630506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017416.630508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017416.630511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344256:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017416.807961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017417.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017417.809982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017418.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017418.811966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017419.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017419.813990:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017420.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017420.815980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017421.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017421.637826:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.637835:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017421.637840:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.637849:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.637857:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.637864:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017421.637899:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335344384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017421.637931:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.637936:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017421.637938:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.637942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.637945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.637949:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017421.637958:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017421.637969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335344512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017421.637978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.637982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017421.637984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.637988:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.637991:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.637994:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017421.637999:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017421.638009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335344640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017421.638018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.638021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017421.638022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.638026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.638028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.638032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638033:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335344384:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017421.638036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017421.638043:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335344768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017421.638052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.638055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017421.638056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.638060:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.638062:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.638066:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017421.638070:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017421.638077:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335344896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017421.638084:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.638087:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017421.638090:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.638093:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.638096:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.638099:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:2.0:1734017421.638101:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734017421.638103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017421.638111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335345024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017421.638118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017421.638121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1734017421.638122:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59cb100 x1818247335344384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017421.638123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017421.638126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1686 00000100:00080000:0.0:1734017421.638129:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017421.638132:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017421.638136:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017421.638142:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335344384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017421.638143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335345152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017421.638152:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638155:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638160:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335344384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017421.638168:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335344512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017421.638185:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638191:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59c9180 x1818247335344512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017421.638203:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335344512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017421.638211:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638213:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638217:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335344512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017421.638222:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335344640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017421.638235:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59cb800 x1818247335344640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017421.638253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335344640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017421.638260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335344640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017421.638290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335344768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017421.638302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59cad80 x1818247335344768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017421.638357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335344768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017421.638365:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335344768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017421.638375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017421.638391:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59c9500 x1818247335344896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017421.638408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335344896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017421.638423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335344896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017421.638433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017421.638445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59c8000 x1818247335345024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017421.638460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335345024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017421.638468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017421.638478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017421.638490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017421.638496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017421/real 1734017421] req@ffff8800b59ca300 x1818247335345152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017421.638506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335345152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017421.638512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017421.638514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017421.638517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345152:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017421.817973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017422.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017422.819899:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017423.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017423.821996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017424.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017424.823979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017425.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017425.825967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017426.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017426.645843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017426.645853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017426.645858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.645872:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00080000:0.0:1734017426.645882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.645891:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017426.645937:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335345280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017426.645984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017426.645990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017426.645993:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.646000:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00080000:0.0:1734017426.646005:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.646011:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017426.646025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017426.646045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335345408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017426.646052:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335345280:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017426.646060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017426.646065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017426.646067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.646072:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00080000:0.0:1734017426.646076:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.646082:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017426.646088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017426.646100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335345536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017426.646112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017426.646116:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017426.646118:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.646123:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00080000:0.0:1734017426.646127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.646133:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017426.646139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734017426.646141:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017426.646153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335345664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017426.646165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734017426.646168:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59ca680 x1818247335345280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017426.646170:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017426.646173:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.646180:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00080000:0.0:1734017426.646184:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.646189:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734017426.646193:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335345280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734017426.646195:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017426.646207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335345792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017426.646208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646213:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335345280:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017426.646253:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017426.646259:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017426.646261:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.646268:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00080000:0.0:1734017426.646273:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.646278:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017426.646285:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017426.646287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335345408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017426.646299:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335345920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017426.646313:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1734017426.646317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017426.646320:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017426.646322:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017426.646328:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1691 00000100:00100000:2.0:1734017426.646329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59c8e00 x1818247335345408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017426.646333:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017426.646338:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017426.646345:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017426.646348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335345408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017426.646358:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335346048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017426.646362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646371:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335345408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017426.646386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335345536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017426.646413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017426.646425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59c8a80 x1818247335345536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017426.646443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335345536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017426.646455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335345536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017426.646469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335345664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017426.646489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017426.646496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59c8700 x1818247335345664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017426.646507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335345664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017426.646514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335345664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017426.646524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017426.646538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017426.646544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59c9f80 x1818247335345792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017426.646556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335345792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017426.646563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017426.646589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017426.646601:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017426.646607:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59cb480 x1818247335345920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017426.646617:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335345920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017426.646625:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646631:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335345920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017426.646636:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017426.646648:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017426.646654:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017426/real 1734017426] req@ffff8800b59cbb80 x1818247335346048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017426.646663:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335346048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017426.646670:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017426.646673:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017426.646676:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346048:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017426.827947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017427.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017427.829991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017428.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017428.831981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017429.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017429.833969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017430.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017430.835945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017431.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017431.653828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.653838:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017431.653843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.653856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.653866:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.653875:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017431.653920:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335346176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017431.653956:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.653960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017431.653962:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.653967:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.653970:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.653974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017431.653985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017431.653996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335346304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017431.654008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.654011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017431.654013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.654017:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.654020:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.654023:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017431.654028:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017431.654038:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335346432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017431.654046:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.654049:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017431.654050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.654054:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.654057:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.654060:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654063:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335346176:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017431.654065:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017431.654073:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335346560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017431.654083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.654086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017431.654087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.654091:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.654094:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.654098:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017431.654102:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017431.654110:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335346688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017431.654118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.654121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017431.654122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.654126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.654128:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.654132:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017431.654164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734017431.654171:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017431.654174:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335346816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017431.654183:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017431.654186:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734017431.654187:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017431.654190:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1696 00000100:00080000:0.0:1734017431.654194:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017431.654197:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59c9c00 x1818247335346176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734017431.654202:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017431.654210:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335346944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017431.654219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335346176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654230:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335346176:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017431.654246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335346304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017431.654267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59caa00 x1818247335346304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017431.654285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335346304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654296:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335346304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017431.654306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335346432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017431.654321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59cbb80 x1818247335346432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017431.654337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335346432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335346432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017431.654379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335346560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017431.654391:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654397:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59cb480 x1818247335346560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017431.654408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335346560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335346560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017431.654426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017431.654438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59c9f80 x1818247335346688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017431.654456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335346688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017431.654473:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017431.654485:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654491:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59c8700 x1818247335346816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017431.654501:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335346816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017431.654518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017431.654530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017431.654537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017431/real 1734017431] req@ffff8800b59c8a80 x1818247335346944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017431.654546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335346944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017431.654561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017431.654563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017431.654566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335346944:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017431.837955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017432.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017432.839961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017433.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017433.841968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017434.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017434.843973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017435.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017435.845973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017436.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734017436.661836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017436.661845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734017436.661850:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.661860:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:0.0:1734017436.661869:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017436.661876:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734017436.661911:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335347072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017436.661947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017436.661952:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017436.661955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.661963:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:0.0:1734017436.661968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017436.661974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017436.661986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017436.662005:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335347200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017436.662015:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335347072:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734017436.662021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017436.662025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017436.662027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.662032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:0.0:1734017436.662036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017436.662040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017436.662085:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017436.662102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335347328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017436.662114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017436.662120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1734017436.662121:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734017436.662122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.662127:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:0.0:1734017436.662131:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017436.662136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017436.662143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017436.662145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59c8e00 x1818247335347072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017436.662156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335347456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017436.662169:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734017436.662169:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335347072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017436.662173:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734017436.662176:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.662181:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:2.0:1734017436.662184:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017436.662185:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017436.662188:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734017436.662190:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734017436.662195:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335347072:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734017436.662196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017436.662207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335347584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017436.662219:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734017436.662223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335347200:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734017436.662224:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734017436.662227:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.662232:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:0.0:1734017436.662236:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017436.662241:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017436.662247:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734017436.662250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017436.662259:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335347712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017436.662261:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59ca680 x1818247335347200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017436.662273:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734017436.662278:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1734017436.662279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335347200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017436.662281:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017436.662286:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1701 00000100:00080000:0.0:1734017436.662290:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017436.662295:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734017436.662301:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1734017436.662303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017436.662307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017436.662312:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335347840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734017436.662312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335347200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017436.662321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335347328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017436.662349:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017436.662360:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59ca300 x1818247335347328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017436.662378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335347328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017436.662391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017436.662394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017436.662400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335347328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017436.662409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335347456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017436.662430:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017436.662438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59c8000 x1818247335347456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017436.662449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335347456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017436.662457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017436.662459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017436.662462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335347456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017436.662468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335347584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734017436.662481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017436.662487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59c9500 x1818247335347584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017436.662498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335347584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017436.662505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017436.662507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017436.662510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335347584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017436.662529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335347712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017436.662541:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017436.662547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59cad80 x1818247335347712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017436.662557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335347712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017436.662564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017436.662567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017436.662570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335347712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017436.662575:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335347840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734017436.662586:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017436.662591:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017436/real 1734017436] req@ffff8800b59cb800 x1818247335347840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017436.662602:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335347840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017436.662609:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017436.662611:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017436.662614:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335347840:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017436.847962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017437.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017437.850072:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017438.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017438.851962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017439.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017439.853974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017440.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017440.856027:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017441.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017441.669930:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017441.669938:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017441.669942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.669952:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.669961:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017441.670011:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017441.670047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335347968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017441.670083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017441.670087:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017441.670089:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.670093:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.670097:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017441.670100:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017441.670111:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017441.670121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335348096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017441.670128:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335347968:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017441.670132:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017441.670135:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017441.670137:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.670141:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.670144:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017441.670147:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017441.670151:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017441.670159:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335348224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017441.670167:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017441.670170:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017441.670171:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.670175:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.670177:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017441.670181:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017441.670184:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017441.670194:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335348352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017441.670196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017441.670202:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017441.670205:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017441.670207:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.670211:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.670214:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017441.670217:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86df800 x1818247335347968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017441.670221:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017441.670228:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335348480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017441.670236:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734017441.670237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335347968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017441.670239:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017441.670241:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.670244:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.670246:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017441.670248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017441.670249:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:0.0:1734017441.670252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734017441.670253:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017441.670258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335347968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017441.670260:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335348608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017441.670268:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017441.670271:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017441.670273:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017441.670276:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1706 00000100:00080000:2.0:1734017441.670278:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017441.670281:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348096:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734017441.670286:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017441.670293:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335348736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017441.670299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86ddc00 x1818247335348096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017441.670318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335348096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017441.670326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017441.670328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017441.670332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017441.670338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017441.670351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86dc000 x1818247335348224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017441.670367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335348224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017441.670375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017441.670378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017441.670381:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017441.670387:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017441.670402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86dd180 x1818247335348352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017441.670419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335348352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017441.670426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017441.670428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017441.670431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017441.670437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335348480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017441.670449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86de680 x1818247335348480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017441.670464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335348480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017441.670472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017441.670474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017441.670477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335348480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017441.670482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335348608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017441.670494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86dd500 x1818247335348608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017441.670516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335348608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017441.670524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017441.670526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017441.670529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335348608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017441.670548:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335348736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017441.670561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017441.670567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017441/real 1734017441] req@ffff8800a86df480 x1818247335348736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017441.670577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335348736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017441.670584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017441.670586:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017441.670589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335348736:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017441.858008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017442.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017442.860004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017443.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017443.862018:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017444.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017444.863995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017445.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017445.865987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017446.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017446.677830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.677838:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017446.677843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017446.677852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.677861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.677868:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017446.677932:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335348864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017446.677964:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.677968:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017446.677970:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017446.677975:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.677978:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.677982:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017446.677991:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017446.678002:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335348992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017446.678012:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.678015:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017446.678018:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017446.678022:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.678024:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.678028:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017446.678032:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017446.678040:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335349120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017446.678049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.678052:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017446.678053:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017446.678057:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.678059:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.678062:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017446.678066:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017446.678071:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335348864:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017446.678076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335349248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017446.678085:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.678088:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017446.678089:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017446.678093:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.678095:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.678098:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017446.678102:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017446.678110:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335349376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017446.678119:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.678122:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017446.678123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017446.678127:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.678129:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.678133:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017446.678136:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017446.678137:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017446.678144:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335349504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017446.678151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017446.678154:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017446.678156:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734017446.678157:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86de300 x1818247335348864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017446.678159:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1711 00000100:00080000:2.0:1734017446.678162:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017446.678164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017446.678168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017446.678175:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335349632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017446.678177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335348864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678188:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678192:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678197:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335348864:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017446.678205:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017446.678222:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017446.678228:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86dce00 x1818247335348992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017446.678239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335348992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335348992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017446.678260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335349120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017446.678273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017446.678279:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86dea00 x1818247335349120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017446.678289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335349120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335349120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017446.678330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335349248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017446.678346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017446.678352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86dd880 x1818247335349248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017446.678363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335349248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335349248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017446.678381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335349376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017446.678393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017446.678398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86df100 x1818247335349376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017446.678408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335349376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335349376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017446.678426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335349504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017446.678438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017446.678444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86dc700 x1818247335349504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017446.678459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335349504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335349504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017446.678477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335349632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017446.678488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017446.678494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017446/real 1734017446] req@ffff8800a86dca80 x1818247335349632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017446.678504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335349632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017446.678511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017446.678513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017446.678516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335349632:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017446.867967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017447.213800:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017447.869971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017448.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017448.872026:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017449.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017449.874029:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017450.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017450.875996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017451.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017451.685872:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.685881:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017451.685886:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.685895:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00080000:2.0:1734017451.685904:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.685911:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017451.685945:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335349760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017451.685976:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.685980:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017451.685982:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.685987:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00080000:2.0:1734017451.685990:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.685994:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017451.686004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017451.686014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335349888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017451.686023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.686026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017451.686028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.686032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00080000:2.0:1734017451.686034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.686037:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017451.686042:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017451.686050:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335350016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017451.686059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.686062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017451.686064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.686067:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00080000:2.0:1734017451.686070:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.686078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017451.686082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017451.686092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335350144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017451.686098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335349760:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017451.686101:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.686104:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017451.686106:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.686109:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00080000:2.0:1734017451.686112:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.686115:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017451.686119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017451.686126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335350272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017451.686134:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.686137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017451.686139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.686142:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00080000:2.0:1734017451.686145:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.686149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017451.686153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017451.686160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335350400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017451.686165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017451.686167:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017451.686170:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017451.686172:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017451.686175:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1716 00000100:00100000:0.0:1734017451.686186:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86dfb80 x1818247335349760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017451.686187:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017451.686190:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017451.686194:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017451.686202:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335350528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017451.686205:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335349760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686225:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335349760:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017451.686233:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335349888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017451.686249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017451.686256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86ddf80 x1818247335349888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017451.686267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335349888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335349888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017451.686287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017451.686300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017451.686306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86dc380 x1818247335350016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017451.686316:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335350016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686330:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017451.686355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017451.686371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017451.686378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86dca80 x1818247335350144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017451.686388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335350144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017451.686406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335350272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017451.686417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017451.686423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86dc700 x1818247335350272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017451.686433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335350272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335350272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017451.686451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335350400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017451.686462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017451.686468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86df100 x1818247335350400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017451.686485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335350400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335350400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017451.686503:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335350528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017451.686515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017451.686521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017451/real 1734017451] req@ffff8800a86dd880 x1818247335350528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017451.686531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335350528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017451.686538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017451.686540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017451.686543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335350528:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017451.878010:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017452.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017452.880001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017453.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017453.881978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017454.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017454.883978:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017455.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017455.885967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017456.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017456.693815:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.693823:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017456.693828:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.693837:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00080000:2.0:1734017456.693846:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.693853:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017456.693887:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335350656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017456.693920:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.693924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017456.693926:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.693930:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00080000:2.0:1734017456.693933:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.693937:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017456.693946:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017456.693956:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335350784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017456.693966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.693969:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017456.693970:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.693974:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00080000:2.0:1734017456.693977:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.693980:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017456.693984:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017456.693992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335350912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017456.694000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.694003:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017456.694004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.694008:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00080000:2.0:1734017456.694011:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.694014:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017456.694018:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017456.694027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335351040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017456.694031:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335350656:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017456.694036:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.694039:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017456.694040:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.694044:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00080000:2.0:1734017456.694046:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.694049:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017456.694053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017456.694060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335351168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017456.694068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.694071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017456.694073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.694076:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00080000:2.0:1734017456.694079:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.694082:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017456.694086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017456.694092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335351296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017456.694099:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017456.694101:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017456.694104:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017456.694106:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017456.694109:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1721 00000100:00100000:0.0:1734017456.694120:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86dea00 x1818247335350656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017456.694121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017456.694125:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017456.694129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017456.694136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335351424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017456.694139:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335350656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694150:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694154:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694159:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335350656:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017456.694166:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017456.694182:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017456.694189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86dce00 x1818247335350784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017456.694200:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335350784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694207:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694210:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694213:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017456.694219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017456.694232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017456.694238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86de300 x1818247335350912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017456.694249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335350912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335350912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017456.694287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017456.694303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017456.694309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86df480 x1818247335351040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017456.694320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335351040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017456.694337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335351168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017456.694349:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017456.694355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86dd500 x1818247335351168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017456.694364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335351168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335351168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017456.694382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335351296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017456.694393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017456.694399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86de680 x1818247335351296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017456.694416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335351296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694424:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335351296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017456.694433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335351424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017456.694445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017456.694451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017456/real 1734017456] req@ffff8800a86dd180 x1818247335351424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017456.694460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335351424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017456.694467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017456.694469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017456.694472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335351424:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017456.887936:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017457.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017457.889968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017458.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017458.891958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017459.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017459.893990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017460.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017460.895935:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017461.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017461.701833:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.701842:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017461.701846:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.701855:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.701864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017461.701872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017461.701907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335351552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017461.701939:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.701943:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017461.701945:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.701950:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.701953:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017461.701957:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017461.701967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017461.701977:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335351680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017461.701986:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.701990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017461.701992:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.701995:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.701998:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017461.702001:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017461.702005:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017461.702013:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335351808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017461.702022:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.702024:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017461.702026:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.702030:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.702032:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017461.702040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017461.702044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017461.702054:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335351936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017461.702055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335351552:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017461.702063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.702066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017461.702068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.702072:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.702074:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017461.702077:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017461.702081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017461.702088:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335352064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017461.702096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.702099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017461.702100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.702103:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.702107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017461.702110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017461.702114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017461.702121:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017461.702121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335352192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017461.702129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017461.702131:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017461.702133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017461.702136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1726 00000100:00080000:2.0:1734017461.702139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017461.702143:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86dc000 x1818247335351552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017461.702143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017461.702147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017461.702154:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335352320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017461.702162:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335351552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702172:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702176:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702181:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335351552:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017461.702188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017461.702206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017461.702212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86ddc00 x1818247335351680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017461.702223:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335351680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702237:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017461.702243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017461.702256:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017461.702262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86df800 x1818247335351808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017461.702272:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335351808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017461.702313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017461.702329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017461.702336:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86dd180 x1818247335351936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017461.702346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335351936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702354:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335351936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017461.702365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017461.702376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017461.702382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86de680 x1818247335352064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017461.702392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335352064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702404:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017461.702410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017461.702421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017461.702427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86dd500 x1818247335352192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017461.702443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335352192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017461.702461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017461.702473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017461.702479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017461/real 1734017461] req@ffff8800a86df480 x1818247335352320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017461.702488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335352320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017461.702496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017461.702498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017461.702500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352320:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017461.897945:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017462.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017462.899879:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017463.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017463.901993:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017464.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017464.903966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017465.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017465.905990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017466.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017466.709847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017466.709856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017466.709861:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.709870:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.709879:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.709886:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017466.709920:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335352448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017466.709952:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017466.709957:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017466.709959:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.709963:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.709966:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.709970:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017466.709979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017466.709991:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335352576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017466.710000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017466.710004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017466.710006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.710009:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.710012:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.710015:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017466.710019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017466.710027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335352704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017466.710035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017466.710038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017466.710039:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.710043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.710045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.710049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017466.710052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017466.710060:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335352448:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017466.710062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335352832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017466.710070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017466.710073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017466.710075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.710079:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.710081:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.710085:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017466.710089:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017466.710096:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335352960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017466.710103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017466.710106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017466.710108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.710111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.710123:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.710127:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:0.0:1734017466.710129:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017466.710131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017466.710138:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335353088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017466.710146:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734017466.710149:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86de300 x1818247335352448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017466.710149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017466.710151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017466.710154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1731 00000100:00080000:2.0:1734017466.710157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017466.710160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017466.710164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017466.710169:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335352448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017466.710171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335353216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017466.710179:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710184:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710189:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335352448:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017466.710196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335352576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017466.710213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017466.710220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86dce00 x1818247335352576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017466.710231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335352576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017466.710239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335352576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017466.710251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335352704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017466.710265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017466.710270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86dea00 x1818247335352704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017466.710280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335352704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017466.710288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335352704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017466.710321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335352832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017466.710337:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017466.710343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86dd880 x1818247335352832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017466.710354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335352832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017466.710362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710364:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710367:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335352832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017466.710372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017466.710383:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017466.710389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86df100 x1818247335352960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017466.710399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335352960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017466.710406:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710408:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335352960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017466.710417:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017466.710428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017466.710434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86dc700 x1818247335353088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017466.710451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335353088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017466.710459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017466.710469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017466.710480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017466.710486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017466/real 1734017466] req@ffff8800a86dca80 x1818247335353216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017466.710496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335353216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017466.710503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017466.710505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017466.710508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353216:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017466.907981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017467.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017467.909955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017468.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017468.911974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017469.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017469.913947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017470.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017470.915946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017471.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017471.717818:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.717829:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017471.717834:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.717847:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.717858:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.717867:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017471.717914:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335353344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017471.717952:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.717957:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017471.717959:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.717964:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.717968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.717972:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017471.717983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017471.717995:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335353472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017471.718006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.718009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017471.718012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.718016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.718019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.718023:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017471.718028:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017471.718037:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335353600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017471.718045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.718048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017471.718050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.718053:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.718056:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.718059:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017471.718063:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017471.718075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335353728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017471.718076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335353344:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017471.718085:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.718088:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017471.718090:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.718094:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.718096:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.718100:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017471.718104:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017471.718113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335353856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017471.718122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.718125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017471.718127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.718130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.718133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.718137:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017471.718141:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017471.718149:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335353984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017471.718156:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017471.718159:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017471.718161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017471.718164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1736 00000100:00080000:2.0:1734017471.718167:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017471.718170:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017471.718174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017471.718175:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335353472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017471.718183:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335354112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017471.718199:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335353600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017471.718226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335353728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017471.718242:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017471.718259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017471.718276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335354112:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017471.917968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017472.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017472.919951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017473.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017473.921994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017474.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734017474.723976:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734017474.724020:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734017474.724039:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017474.724073:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017474.724080:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017474.724087:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017474.724093:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017474.724100:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734017474.724105:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:2.0:1734017474.724177:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86dca80 x1818247335353728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724214:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335353728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335353728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017474.724243:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86dc700 x1818247335353856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724256:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335353856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724266:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017474.724276:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86df100 x1818247335353984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335353984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724296:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335353984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017474.724306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86dd880 x1818247335354112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335354112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335354112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017474.724352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86dc380 x1818247335353344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335353344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335353344:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017474.724382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86ddf80 x1818247335353472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335353472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335353472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017474.724483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017471/real 1734017474] req@ffff8800a86dfb80 x1818247335353600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017474.724510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335353600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017474.724519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017474.724521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017474.724524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335353600:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017474.923936:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017475.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017475.926001:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017476.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017476.725834:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.725846:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017476.725852:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017476.725865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.725876:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.725885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017476.725930:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335354240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017476.725969:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.725973:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017476.725975:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017476.725980:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.725984:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.725987:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017476.725998:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017476.726011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335354368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017476.726021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.726025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017476.726026:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017476.726030:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.726032:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.726036:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017476.726041:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017476.726049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335354496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017476.726057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.726061:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017476.726062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017476.726066:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.726069:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.726072:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017476.726076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017476.726085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335354624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017476.726094:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.726098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1734017476.726099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335354240:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017476.726100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017476.726105:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.726108:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.726112:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017476.726117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017476.726125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335354752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017476.726133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.726136:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017476.726137:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017476.726141:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.726143:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.726147:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017476.726151:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017476.726158:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335354880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017476.726166:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017476.726169:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017476.726170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1734017476.726173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017476.726174:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1741 00000100:00080000:2.0:1734017476.726178:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017476.726181:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017476.726185:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017476.726193:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86dfb80 x1818247335354240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017476.726193:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335355008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017476.726213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335354240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335354240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017476.726241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335354368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017476.726258:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017476.726265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86ddf80 x1818247335354368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017476.726276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335354368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726284:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335354368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017476.726297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335354496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017476.726312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017476.726318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86dc380 x1818247335354496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017476.726329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335354496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335354496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017476.726427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335354624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017476.726446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017476.726454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86dd880 x1818247335354624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017476.726464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335354624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335354624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017476.726484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335354752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017476.726496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017476.726502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86df100 x1818247335354752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017476.726519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335354752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335354752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017476.726538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335354880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017476.726549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017476.726555:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86dc700 x1818247335354880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017476.726565:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335354880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726572:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726577:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335354880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017476.726583:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017476.726595:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017476.726601:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017476/real 1734017476] req@ffff8800a86dca80 x1818247335355008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017476.726611:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335355008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017476.726618:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017476.726621:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017476.726625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355008:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017476.927942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017477.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017477.929975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017478.213938:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017478.931888:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017479.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017479.933938:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017480.213782:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017480.935939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017481.213786:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017481.733850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.733861:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017481.733866:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017481.733878:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.733887:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.733896:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017481.733932:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335355136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017481.733968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.733972:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017481.733974:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017481.733978:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.733981:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.733985:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017481.733996:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017481.734006:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335355264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017481.734016:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.734019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017481.734021:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017481.734024:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.734027:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.734032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017481.734036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017481.734044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335355392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017481.734052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.734055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017481.734057:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017481.734060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.734062:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.734065:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017481.734069:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017481.734080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335355520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017481.734085:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335355136:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017481.734089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.734092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017481.734094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017481.734098:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.734100:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.734103:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017481.734107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017481.734114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335355648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017481.734122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.734125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017481.734127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017481.734131:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.734133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.734136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017481.734140:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017481.734147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335355776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017481.734154:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017481.734157:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017481.734158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1734017481.734161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017481.734162:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1746 00000100:00080000:2.0:1734017481.734165:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017481.734168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017481.734172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017481.734179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335355904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017481.734183:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86dea00 x1818247335355136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734196:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335355136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734204:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734207:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734210:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335355136:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017481.734215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335355264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017481.734227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017481.734231:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86dce00 x1818247335355264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335355264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734242:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335355264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017481.734251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335355392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017481.734260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017481.734263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86de300 x1818247335355392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335355392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335355392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017481.734331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335355520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017481.734341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017481.734345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86df480 x1818247335355520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335355520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335355520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017481.734362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017481.734369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017481.734373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86dd500 x1818247335355648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335355648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017481.734390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017481.734396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017481.734399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86de680 x1818247335355776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734404:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335355776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734410:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017481.734414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017481.734421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017481.734424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017481/real 1734017481] req@ffff8800a86dd180 x1818247335355904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017481.734429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335355904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017481.734438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017481.734440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017481.734442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335355904:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017481.938021:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017482.213777:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017482.939934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017483.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017483.942016:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017484.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017484.943971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017485.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017485.945968:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017486.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017486.741865:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017486.741876:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017486.741882:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.741894:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1751 00000100:00080000:2.0:1734017486.741904:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.741914:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017486.741960:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335356032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017486.742000:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017486.742007:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017486.742010:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.742016:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1751 00000100:00080000:2.0:1734017486.742022:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.742028:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017486.742043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017486.742064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335356160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017486.742070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335356032:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017486.742080:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017486.742084:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017486.742086:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.742092:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1751 00000100:00080000:2.0:1734017486.742097:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.742102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017486.742108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017486.742121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335356288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017486.742133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017486.742137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017486.742139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.742144:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1751 00000100:00080000:2.0:1734017486.742149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.742155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017486.742162:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017486.742163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017486.742178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335356416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017486.742188:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86df800 x1818247335356032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017486.742190:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017486.742230:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017486.742232:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.742239:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1751 00000100:00080000:2.0:1734017486.742244:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.742251:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017486.742258:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017486.742266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335356032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017486.742273:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335356544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017486.742282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734017486.742287:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734017486.742293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335356032:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017486.742293:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017486.742295:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.742301:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1751 00000100:00080000:2.0:1734017486.742305:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.742310:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017486.742315:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017486.742327:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335356672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017486.742330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335356160:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734017486.742341:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017486.742347:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017486.742349:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017486.742354:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1751 00000800:00000100:0.0:1734017486.742359:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017486.742359:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017486.742364:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017486.742371:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017486.742372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86ddc00 x1818247335356160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017486.742384:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335356800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017486.742390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335356160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017486.742415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017486.742424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335356160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017486.742433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335356288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017486.742463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017486.742476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86dc000 x1818247335356288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017486.742488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335356288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017486.742497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017486.742502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335356288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017486.742509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335356416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017486.742525:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017486.742532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86dd180 x1818247335356416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017486.742543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335356416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017486.742550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017486.742556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335356416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017486.742562:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335356544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017486.742576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017486.742582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86de680 x1818247335356544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017486.742593:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335356544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017486.742601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017486.742606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335356544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017486.742624:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335356672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017486.742637:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017486.742642:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86dd500 x1818247335356672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017486.742652:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335356672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017486.742660:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742663:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017486.742666:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335356672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017486.742671:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335356800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017486.742683:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017486.742688:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017486/real 1734017486] req@ffff8800a86df480 x1818247335356800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017486.742738:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335356800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017486.742747:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017486.742750:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017486.742753:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335356800:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017486.947938:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017487.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017487.949968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017488.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017488.951921:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017489.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017489.953978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017490.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017490.955981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017491.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017491.749850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.749859:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017491.749863:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.749873:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.749882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.749890:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017491.749927:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335356928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017491.749959:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.749963:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017491.749966:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.749970:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.749973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.749977:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017491.749986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017491.749997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335357056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017491.750007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.750010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017491.750012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.750015:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.750018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.750021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017491.750026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017491.750033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335357184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017491.750041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.750044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017491.750046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.750049:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.750051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.750055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017491.750059:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017491.750067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335356928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017491.750068:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335357312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017491.750077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.750080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017491.750082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.750085:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.750088:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.750092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017491.750096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017491.750103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335357440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017491.750146:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.750150:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017491.750153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.750157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.750164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.750168:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017491.750174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017491.750175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017491.750184:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335357568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017491.750193:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017491.750196:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1734017491.750197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86de300 x1818247335356928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017491.750197:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017491.750201:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1756 00000100:00080000:2.0:1734017491.750204:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017491.750207:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017491.750211:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017491.750217:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335356928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017491.750218:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335357696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017491.750227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335356928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017491.750243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017491.750263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017491.750270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86dce00 x1818247335357056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017491.750289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335357056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017491.750297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017491.750308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017491.750327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017491.750333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86dea00 x1818247335357184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017491.750343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335357184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017491.750350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017491.750381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017491.750394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017491.750400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86dca80 x1818247335357312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017491.750410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335357312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017491.750417:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750421:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017491.750430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335357440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017491.750442:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017491.750448:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86dc700 x1818247335357440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017491.750458:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335357440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017491.750465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335357440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017491.750476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335357568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017491.750488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017491.750493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86df100 x1818247335357568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017491.750503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335357568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017491.750511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335357568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017491.750521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335357696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017491.750536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017491.750542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017491/real 1734017491] req@ffff8800a86dd880 x1818247335357696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017491.750552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335357696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017491.750560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017491.750562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017491.750565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335357696:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017491.957971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017492.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017492.959961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017493.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017493.962008:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017494.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017494.963957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017495.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017495.965942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017496.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017496.757830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.757839:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017496.757843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.757852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.757861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017496.757868:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017496.757902:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335357824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017496.757934:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.757938:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017496.757940:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.757944:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.757947:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017496.757951:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017496.757960:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017496.757971:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335357952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017496.757980:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.757983:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017496.757985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.757989:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.757991:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017496.757995:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017496.758000:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017496.758008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335358080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017496.758016:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.758019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017496.758021:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.758054:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.758058:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017496.758062:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017496.758066:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017496.758077:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335358208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017496.758084:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335357824:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017496.758087:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.758090:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017496.758092:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.758095:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.758098:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017496.758101:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017496.758105:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017496.758113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335358336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017496.758122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.758124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017496.758127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.758130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.758133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017496.758136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017496.758140:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017496.758149:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335358464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017496.758150:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017496.758157:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017496.758159:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017496.758161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017496.758164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1761 00000100:00080000:2.0:1734017496.758167:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017496.758170:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86dc380 x1818247335357824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017496.758170:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017496.758174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017496.758181:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335358592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017496.758189:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335357824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758199:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758203:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335357824:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017496.758215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017496.758235:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017496.758242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86ddf80 x1818247335357952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017496.758253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335357952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335357952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017496.758272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017496.758286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017496.758291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86dfb80 x1818247335358080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017496.758302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335358080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017496.758339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017496.758351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017496.758358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86dd880 x1818247335358208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017496.758375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335358208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017496.758394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335358336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017496.758406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017496.758412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86df100 x1818247335358336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017496.758421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335358336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335358336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017496.758440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335358464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017496.758451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017496.758458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86dc700 x1818247335358464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017496.758467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335358464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335358464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017496.758486:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335358592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017496.758499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017496.758504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017496/real 1734017496] req@ffff8800a86dca80 x1818247335358592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017496.758514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335358592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017496.758521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017496.758523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017496.758526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335358592:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017496.967950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017497.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017497.969996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017498.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017498.971924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017499.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017499.973963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017500.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017500.975946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017501.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017501.765839:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.765848:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017501.765852:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017501.765862:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.765870:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.765878:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017501.765913:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335358720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017501.765989:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.765993:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017501.765995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017501.766000:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.766004:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.766008:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017501.766018:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017501.766029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335358848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017501.766039:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.766043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017501.766044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017501.766047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.766050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.766053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017501.766058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017501.766066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335358976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017501.766074:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.766077:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017501.766078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017501.766082:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.766084:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.766089:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017501.766093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017501.766099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335358720:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017501.766103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335359104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017501.766112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.766115:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017501.766117:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017501.766120:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.766123:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.766126:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017501.766130:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017501.766138:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335359232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017501.766145:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.766148:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017501.766150:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017501.766153:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.766155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.766159:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017501.766162:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017501.766165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017501.766169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335359360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017501.766177:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017501.766181:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017501.766182:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734017501.766186:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86dea00 x1818247335358720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017501.766186:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1766 00000100:00080000:2.0:1734017501.766189:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017501.766192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017501.766196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017501.766204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335359488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017501.766206:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335358720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766217:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335358720:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017501.766234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017501.766250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017501.766257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86dce00 x1818247335358848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017501.766268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335358848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017501.766289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017501.766302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017501.766308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86de300 x1818247335358976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017501.766318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335358976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335358976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017501.766358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335359104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017501.766373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017501.766380:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86df480 x1818247335359104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017501.766390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335359104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335359104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017501.766409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335359232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017501.766420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017501.766426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86dd500 x1818247335359232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017501.766436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335359232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335359232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017501.766453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335359360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017501.766465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017501.766471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86de680 x1818247335359360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017501.766487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335359360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335359360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017501.766505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335359488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017501.766517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017501.766523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017501/real 1734017501] req@ffff8800a86dd180 x1818247335359488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017501.766532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335359488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017501.766539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017501.766542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017501.766545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335359488:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017501.978020:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017502.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017502.979997:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017503.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017503.982005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017504.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017504.983937:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017505.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017505.985956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017506.213939:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017506.773848:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.773895:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017506.773899:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.773909:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00080000:2.0:1734017506.773918:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.773925:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017506.773960:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335359616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017506.773991:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.773995:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017506.773997:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.774002:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00080000:2.0:1734017506.774005:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.774008:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017506.774017:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017506.774028:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335359744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017506.774038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.774042:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017506.774043:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.774047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00080000:2.0:1734017506.774050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.774053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017506.774057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017506.774065:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335359872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017506.774073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.774076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017506.774077:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.774081:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00080000:2.0:1734017506.774083:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.774086:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017506.774090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017506.774095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335359616:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017506.774100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335360000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017506.774109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.774112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017506.774113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.774118:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00080000:2.0:1734017506.774120:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.774123:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017506.774127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017506.774134:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335360128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017506.774142:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.774145:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017506.774146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.774150:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00080000:2.0:1734017506.774152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.774156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017506.774159:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017506.774163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017506.774166:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335360256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017506.774174:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017506.774177:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017506.774178:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017506.774182:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1771 00000100:00100000:0.0:1734017506.774183:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86dc000 x1818247335359616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017506.774185:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017506.774187:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017506.774191:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017506.774199:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335360384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017506.774211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335359616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774221:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774225:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774229:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335359616:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017506.774236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335359744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017506.774254:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017506.774260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86ddc00 x1818247335359744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017506.774271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335359744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774279:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335359744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017506.774290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335359872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017506.774309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017506.774315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86df800 x1818247335359872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017506.774325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335359872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335359872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017506.774364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017506.774377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017506.774383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86dd180 x1818247335360000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017506.774393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335360000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017506.774411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335360128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017506.774423:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017506.774429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86de680 x1818247335360128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017506.774438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335360128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335360128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017506.774456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335360256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017506.774471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017506.774477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86dd500 x1818247335360256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017506.774487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335360256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335360256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017506.774504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335360384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017506.774516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017506.774522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017506/real 1734017506] req@ffff8800a86df480 x1818247335360384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017506.774532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335360384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017506.774539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017506.774541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017506.774544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335360384:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017506.987935:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017507.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017507.990010:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017508.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017508.992017:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017509.213883:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017509.994019:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017510.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017510.995992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017511.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017511.781857:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.781866:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017511.781870:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.781880:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.781889:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.781897:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017511.781933:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335360512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017511.781964:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.781968:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017511.781970:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.781974:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.781978:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.781981:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017511.781991:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017511.782001:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335360640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017511.782010:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.782013:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017511.782016:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.782020:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.782022:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.782026:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017511.782030:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017511.782038:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335360768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017511.782046:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.782049:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017511.782051:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.782054:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.782056:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.782060:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017511.782064:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017511.782074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335360512:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017511.782074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335360896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017511.782083:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.782086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017511.782087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.782091:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.782093:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.782097:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017511.782101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017511.782108:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335361024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017511.782117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.782120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017511.782121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.782125:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.782127:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.782131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017511.782134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017511.782141:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335361152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017511.782145:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017511.782149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017511.782152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017511.782153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017511.782156:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1776 00000100:00080000:2.0:1734017511.782160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017511.782164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86de300 x1818247335360512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017511.782168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017511.782176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335361280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017511.782192:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335360512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782211:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335360512:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017511.782218:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017511.782235:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86dce00 x1818247335360640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017511.782253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335360640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017511.782272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017511.782286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86dea00 x1818247335360768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017511.782302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335360768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017511.782342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017511.782354:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782361:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86dca80 x1818247335360896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017511.782371:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335360896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782378:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782381:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335360896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017511.782388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017511.782400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86dc700 x1818247335361024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017511.782415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335361024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017511.782433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017511.782448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86df100 x1818247335361152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017511.782464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335361152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017511.782481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017511.782492:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017511.782498:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017511/real 1734017511] req@ffff8800a86dd880 x1818247335361280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017511.782508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335361280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017511.782516:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017511.782518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017511.782521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361280:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017511.998009:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017512.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017512.999987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017513.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017514.001984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017514.213932:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017515.003972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017515.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017516.005971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017516.213788:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017516.789849:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.789857:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017516.789861:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017516.789871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.789880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.789887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017516.789922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335361408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017516.789955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.789959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017516.789961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017516.789965:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.789968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.789972:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017516.789982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017516.789992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335361536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017516.790001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.790004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017516.790006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017516.790010:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.790012:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.790015:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017516.790020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017516.790028:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335361664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017516.790036:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.790040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017516.790041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017516.790045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.790047:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.790050:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017516.790054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017516.790060:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335361408:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017516.790065:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335361792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017516.790073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.790076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017516.790078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017516.790081:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.790083:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.790087:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017516.790091:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017516.790098:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335361920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017516.790107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.790109:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017516.790111:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017516.790114:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.790118:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.790121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017516.790125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017516.790127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017516.790133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335362048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017516.790141:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017516.790144:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017516.790146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734017516.790148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86dfb80 x1818247335361408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017516.790149:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1781 00000100:00080000:2.0:1734017516.790152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017516.790155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017516.790159:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017516.790166:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335362176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017516.790168:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335361408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790179:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790182:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790187:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335361408:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017516.790194:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335361536:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017516.790211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017516.790218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86ddf80 x1818247335361536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017516.790229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335361536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790236:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790239:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335361536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017516.790247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335361664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017516.790265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017516.790271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86dc380 x1818247335361664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017516.790281:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335361664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335361664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017516.790328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335361792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017516.790340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017516.790346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86dd880 x1818247335361792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017516.790357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335361792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790365:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335361792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017516.790375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017516.790387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017516.790393:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86df100 x1818247335361920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017516.790402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335361920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335361920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017516.790422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017516.790433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017516.790439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86dc700 x1818247335362048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017516.790449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335362048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790461:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017516.790466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017516.790478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017516.790484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017516/real 1734017516] req@ffff8800a86dca80 x1818247335362176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017516.790494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335362176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017516.790501:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017516.790503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017516.790506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362176:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017517.007959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017517.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017518.009956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017518.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017519.011954:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017519.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017520.013996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017520.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017521.016003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017521.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017521.797825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017521.797837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017521.797843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.797856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.797866:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017521.797876:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017521.797924:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335362304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017521.797961:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017521.797965:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017521.797967:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.797972:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.797976:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017521.797980:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017521.797992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017521.798004:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335362432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017521.798015:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335362304:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017521.798015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017521.798018:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017521.798020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.798024:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.798027:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017521.798031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017521.798035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017521.798044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335362560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017521.798052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017521.798055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017521.798057:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.798060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.798063:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017521.798067:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017521.798071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017521.798082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335362688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017521.798097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1734017521.798099:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017521.798101:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017521.798103:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.798107:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.798110:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017521.798113:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017521.798117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734017521.798123:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86dea00 x1818247335362304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017521.798126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335362816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017521.798133:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017521.798137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017521.798138:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.798142:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.798144:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017521.798145:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335362304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017521.798148:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017521.798152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1734017521.798157:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798160:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017521.798160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335362944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017521.798166:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335362304:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017521.798168:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017521.798171:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017521.798173:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017521.798177:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1786 00000100:00080000:2.0:1734017521.798180:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017521.798184:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017521.798188:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017521.798194:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335362432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017521.798196:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335363072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017521.798213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017521.798220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86dce00 x1818247335362432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017521.798232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335362432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017521.798240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017521.798246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335362432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017521.798252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335362560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017521.798267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017521.798273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86de300 x1818247335362560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017521.798292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335362560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017521.798300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017521.798305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335362560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017521.798310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335362688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017521.798323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017521.798329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86df480 x1818247335362688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017521.798338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335362688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017521.798346:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017521.798351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335362688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017521.798357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017521.798370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017521.798376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86dd500 x1818247335362816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017521.798386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335362816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017521.798393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017521.798399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017521.798405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017521.798418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017521.798423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86de680 x1818247335362944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017521.798433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335362944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017521.798440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017521.798446:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335362944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017521.798464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017521.798477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017521.798483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017521/real 1734017521] req@ffff8800a86dd180 x1818247335363072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017521.798493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335363072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017521.798500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017521.798502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017521.798505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363072:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017522.017992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017522.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017523.019971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017523.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017524.021952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017524.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017525.023980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017525.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017526.025980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017526.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017526.805840:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.805848:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017526.805853:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017526.805863:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.805873:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.805881:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017526.805917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335363200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017526.805952:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.805956:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017526.805958:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017526.805963:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.805966:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.805969:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017526.805979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017526.805990:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335363328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017526.805999:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.806003:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017526.806005:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017526.806009:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.806012:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.806015:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017526.806019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017526.806027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335363456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017526.806036:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.806039:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017526.806041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017526.806044:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.806046:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.806050:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017526.806054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017526.806064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335363584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017526.806074:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.806077:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017526.806078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734017526.806079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335363200:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017526.806082:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.806084:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.806088:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017526.806093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017526.806101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335363712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017526.806110:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.806113:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017526.806115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017526.806118:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.806120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.806123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017526.806127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017526.806134:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335363840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017526.806142:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017526.806145:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017526.806146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017526.806149:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1791 00000100:00080000:2.0:1734017526.806154:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017526.806157:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017526.806161:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017526.806165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017526.806169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335363968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017526.806191:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86df800 x1818247335363200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335363200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335363200:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017526.806242:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335363328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017526.806260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017526.806267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86ddc00 x1818247335363328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335363328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335363328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017526.806300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335363456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017526.806314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017526.806322:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86dc000 x1818247335363456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335363456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335363456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017526.806377:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335363584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017526.806389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017526.806395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86dd180 x1818247335363584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335363584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806418:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335363584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017526.806425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017526.806438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017526.806444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86de680 x1818247335363712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335363712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017526.806481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017526.806493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017526.806499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86dd500 x1818247335363840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335363840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806568:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017526.806574:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017526.806591:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017526.806598:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017526/real 1734017526] req@ffff8800a86df480 x1818247335363968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017526.806608:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335363968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017526.806616:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017526.806619:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017526.806622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335363968:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017527.027967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017527.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017528.029966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017528.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017529.031936:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017529.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017530.033973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017530.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017531.035972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017531.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017531.813839:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.813847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017531.813851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.813861:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.813870:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.813878:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017531.813914:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335364096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017531.813949:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.813953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017531.813955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.813959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.813962:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.813966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017531.813975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017531.813986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335364224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017531.813996:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.813999:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017531.814001:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.814004:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.814007:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.814010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017531.814015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017531.814023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335364352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017531.814031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.814034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017531.814037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.814041:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.814043:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.814046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017531.814050:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017531.814059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335364480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017531.814069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.814072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017531.814073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.814077:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.814080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.814083:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814087:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335364096:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734017531.814087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017531.814095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335364608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017531.814103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.814106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017531.814107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.814111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.814113:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.814117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017531.814121:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017531.814128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335364736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017531.814137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017531.814140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017531.814142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017531.814145:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1796 00000100:00080000:2.0:1734017531.814147:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017531.814150:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017531.814154:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017531.814161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335364864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017531.814181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86de300 x1818247335364096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335364096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335364096:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017531.814258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335364224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017531.814276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86dce00 x1818247335364224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335364224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335364224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017531.814321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335364352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017531.814336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86dea00 x1818247335364352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335364352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335364352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017531.814400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335364480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017531.814412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86dca80 x1818247335364480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335364480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335364480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017531.814499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335364608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017531.814516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86dc700 x1818247335364608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335364608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335364608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017531.814554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335364736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017531.814567:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86df100 x1818247335364736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335364736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814590:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335364736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017531.814601:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335364864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017531.814615:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017531.814621:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017531/real 1734017531] req@ffff8800a86dd880 x1818247335364864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017531.814631:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335364864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017531.814639:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017531.814643:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017531.814646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335364864:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017532.037943:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017532.213905:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017533.039950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017533.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017534.042174:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017534.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017535.043931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017535.213941:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017536.045983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017536.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017536.821840:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.821850:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017536.821854:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.821865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.821874:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.821881:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017536.821917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335364992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017536.821949:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.821953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017536.821955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.821960:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.821963:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.821967:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017536.821976:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017536.821987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335365120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017536.821997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.822000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017536.822001:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.822005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.822008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.822011:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017536.822015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017536.822023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335365248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017536.822031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.822034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017536.822035:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.822039:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.822041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.822044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017536.822048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017536.822058:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335365376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017536.822059:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335364992:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017536.822073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.822076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017536.822077:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.822081:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.822083:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.822087:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017536.822090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017536.822098:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335365504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017536.822105:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.822108:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017536.822110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.822113:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.822117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.822120:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017536.822124:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017536.822130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335365632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017536.822138:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017536.822141:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017536.822142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017536.822146:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1801 00000100:00080000:2.0:1734017536.822149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017536.822152:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734017536.822154:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335365120:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734017536.822156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017536.822164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335365760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017536.822176:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335365248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017536.822199:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335365376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017536.822213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335365504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017536.822226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335365632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017536.822240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335365760:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017537.047948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017537.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017538.049941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017538.213925:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017539.051964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017539.213925:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:3.0:1734017539.827984:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:3.0:1734017539.828030:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1734017539.828049:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:3.0:1734017539.828084:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:3.0:1734017539.828094:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:3.0:1734017539.828100:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:3.0:1734017539.828107:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:3.0:1734017539.828113:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:3.0:1734017539.828119:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734017539.828203:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86dd880 x1818247335365376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335365376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335365376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017539.828277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86df100 x1818247335365504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335365504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335365504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017539.828392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86dc700 x1818247335365632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335365632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335365632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017539.828443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86dca80 x1818247335365760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335365760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335365760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017539.828505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86dc380 x1818247335364992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335364992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335364992:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017539.828566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86ddf80 x1818247335365120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335365120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828595:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828598:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828604:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335365120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017539.828613:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017536/real 1734017539] req@ffff8800a86dfb80 x1818247335365248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017539.828630:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335365248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017539.828642:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017539.828646:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017539.828652:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335365248:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017540.053924:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017540.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017541.056001:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017541.213800:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017541.829845:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.829856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017541.829861:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.829874:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00080000:2.0:1734017541.829885:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.829895:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017541.829943:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335365888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017541.829981:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.829985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017541.829987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.829992:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00080000:2.0:1734017541.829996:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.829999:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017541.830011:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017541.830022:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335366016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017541.830032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.830036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017541.830038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.830042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00080000:2.0:1734017541.830045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.830049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017541.830053:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017541.830062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335366144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017541.830070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.830074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017541.830075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.830079:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00080000:2.0:1734017541.830081:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.830084:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017541.830089:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017541.830093:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335365888:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017541.830100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335366272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017541.830109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.830112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017541.830113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.830117:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00080000:2.0:1734017541.830120:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.830123:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017541.830129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017541.830137:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335366400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017541.830145:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.830148:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017541.830150:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.830153:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00080000:2.0:1734017541.830156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.830160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017541.830164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017541.830171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335366528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017541.830175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017541.830179:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017541.830182:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017541.830183:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017541.830187:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1806 00000100:00100000:0.0:1734017541.830198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86dea00 x1818247335365888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017541.830199:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017541.830203:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017541.830207:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017541.830215:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335366656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017541.830219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335365888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830230:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830234:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335365888:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017541.830246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017541.830306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017541.830315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86dce00 x1818247335366016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017541.830327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335366016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017541.830349:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017541.830364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017541.830370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86de300 x1818247335366144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017541.830389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335366144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017541.830432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017541.830444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017541.830451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86df480 x1818247335366272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017541.830460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335366272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017541.830478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335366400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017541.830490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017541.830496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86dd500 x1818247335366400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017541.830505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335366400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335366400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017541.830524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335366528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017541.830536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017541.830542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86de680 x1818247335366528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017541.830551:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335366528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335366528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017541.830569:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335366656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017541.830581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017541.830587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017541/real 1734017541] req@ffff8800a86dd180 x1818247335366656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017541.830596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335366656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017541.830603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017541.830605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017541.830608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335366656:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017542.057975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017542.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017543.059956:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017543.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017544.061976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017544.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017545.063960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017545.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017546.065958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017546.213934:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017546.837843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.837854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017546.837858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.837868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.837877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.837885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017546.837920:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335366784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017546.837953:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.837958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017546.837960:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.837964:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.837967:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.837971:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017546.837981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017546.837991:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335366912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017546.838001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.838004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017546.838006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.838010:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.838013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.838016:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017546.838021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017546.838029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335367040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017546.838038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.838041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017546.838043:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.838046:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.838048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.838052:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017546.838056:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017546.838057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335366784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017546.838065:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335367168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017546.838074:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.838077:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017546.838078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.838081:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.838084:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.838087:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017546.838091:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017546.838099:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335367296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017546.838107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.838110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017546.838112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.838115:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.838118:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.838122:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:0.0:1734017546.838123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017546.838126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017546.838133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335367424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017546.838140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017546.838143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1734017546.838145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86dc000 x1818247335366784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017546.838145:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017546.838148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1811 00000100:00080000:2.0:1734017546.838151:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017546.838154:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017546.838157:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017546.838165:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335366784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017546.838165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335367552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017546.838214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838223:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335366784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017546.838231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366912:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017546.838252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017546.838260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86ddc00 x1818247335366912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017546.838272:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335366912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017546.838280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335366912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017546.838291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017546.838304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017546.838310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86df800 x1818247335367040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017546.838322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335367040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017546.838331:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017546.838361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017546.838373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017546.838379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86dd180 x1818247335367168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017546.838389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335367168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017546.838397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017546.838409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335367296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017546.838420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017546.838426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86de680 x1818247335367296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017546.838436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335367296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017546.838444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335367296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017546.838456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335367424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017546.838467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017546.838473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86dd500 x1818247335367424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017546.838483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335367424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017546.838491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335367424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017546.838501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335367552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017546.838513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017546.838519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017546/real 1734017546] req@ffff8800a86df480 x1818247335367552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017546.838528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335367552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017546.838535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017546.838537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017546.838540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335367552:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017547.067951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017547.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017548.069951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017548.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017549.071991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017549.213776:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017550.073963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017550.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017551.075979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017551.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017551.845827:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.845838:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017551.845843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017551.845857:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00080000:2.0:1734017551.845868:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017551.845877:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017551.845926:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335367680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017551.845962:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.845967:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017551.845969:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017551.845973:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00080000:2.0:1734017551.845977:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017551.845980:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017551.845992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017551.846004:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335367808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017551.846014:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.846017:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017551.846019:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017551.846023:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00080000:2.0:1734017551.846025:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017551.846029:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017551.846034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017551.846042:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335367936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017551.846051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.846054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017551.846055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017551.846060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00080000:2.0:1734017551.846063:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017551.846066:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017551.846071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017551.846072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335367680:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017551.846081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335368064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017551.846200:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017551.846224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86de300 x1818247335367680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017551.846245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335367680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017551.846256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017551.846260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017551.846265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335367680:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017551.846273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017551.846290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017551.846297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86dce00 x1818247335367808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017551.846308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335367808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017551.846310:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.846313:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017551.846315:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734017551.846317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017551.846319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734017551.846320:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00100000:0.0:1734017551.846323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367808:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734017551.846324:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017551.846328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367936:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734017551.846329:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017551.846335:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1734017551.846344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017551.846346:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335368192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017551.846350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86dea00 x1818247335367936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017551.846356:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.846359:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1734017551.846360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335367936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017551.846361:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017551.846365:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00080000:2.0:1734017551.846368:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734017551.846369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017551.846371:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:0.0:1734017551.846372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734017551.846375:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017551.846376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335367936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017551.846384:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335368320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017551.846393:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017551.846396:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017551.846398:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017551.846402:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1816 00000100:00080000:2.0:1734017551.846404:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017551.846408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368064:192.168.201.104@tcp:8:kworker.0 00000100:00080000:2.0:1734017551.846408:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017551.846412:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017551.846420:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335368448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017551.846421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017551.846427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86dfb80 x1818247335368064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017551.846438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335368064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017551.846446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017551.846448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017551.846451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017551.846457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335368192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017551.846469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017551.846475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86ddf80 x1818247335368192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017551.846485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335368192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017551.846492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017551.846495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017551.846498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335368192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017551.846506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335368320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017551.846518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017551.846524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86dc380 x1818247335368320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017551.846534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335368320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017551.846542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017551.846544:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017551.846547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335368320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017551.846560:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335368448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017551.846573:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017551.846579:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017551/real 1734017551] req@ffff8800a86dca80 x1818247335368448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017551.846588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335368448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017551.846596:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017551.846599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017551.846602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335368448:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017552.077982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017552.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017553.079952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017553.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017554.081924:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017554.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017555.083962:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017555.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017556.086011:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017556.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017556.853862:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.853873:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017556.853878:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.853892:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00080000:2.0:1734017556.853903:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.853912:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017556.853959:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335368576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017556.853997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.854029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017556.854032:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.854038:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00080000:2.0:1734017556.854042:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.854045:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017556.854058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017556.854071:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335368704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017556.854082:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.854086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017556.854088:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.854092:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00080000:2.0:1734017556.854094:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.854098:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017556.854102:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734017556.854108:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335368576:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017556.854111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335368832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017556.854120:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.854124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017556.854125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.854129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00080000:2.0:1734017556.854131:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.854135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017556.854140:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017556.854150:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335368960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017556.854160:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.854163:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017556.854165:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.854168:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00080000:2.0:1734017556.854171:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.854175:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017556.854179:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017556.854187:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335369088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017556.854195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017556.854195:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.854198:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017556.854200:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.854204:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00080000:2.0:1734017556.854207:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.854211:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017556.854215:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017556.854220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86dc700 x1818247335368576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017556.854223:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335369216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017556.854230:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017556.854234:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017556.854235:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017556.854238:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1821 00000100:00100000:0.0:1734017556.854242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335368576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017556.854242:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017556.854246:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017556.854250:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1734017556.854254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017556.854258:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335369344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017556.854263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335368576:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017556.854270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368704:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017556.854288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017556.854295:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86df100 x1818247335368704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017556.854306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335368704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017556.854314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854317:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017556.854320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017556.854347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017556.854362:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017556.854368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86dd880 x1818247335368832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017556.854378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335368832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017556.854386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017556.854393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017556.854399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017556.854412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017556.854418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86dca80 x1818247335368960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017556.854428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335368960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017556.854436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017556.854442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335368960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017556.854447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017556.854460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017556.854466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86dc380 x1818247335369088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017556.854476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335369088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017556.854483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017556.854488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017556.854494:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017556.854511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017556.854517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86ddf80 x1818247335369216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017556.854528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335369216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017556.854535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017556.854541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017556.854546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017556.854559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017556.854566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017556/real 1734017556] req@ffff8800a86dfb80 x1818247335369344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017556.854576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335369344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017556.854584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017556.854585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017556.854588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369344:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017557.087961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017557.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017558.089946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017558.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017559.092004:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017559.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017560.093917:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017560.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017561.096024:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017561.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017561.861804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.861815:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017561.861820:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.861834:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.861844:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.861853:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017561.861900:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335369472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017561.861978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.861983:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017561.861985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.861991:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.861995:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.861999:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017561.862011:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017561.862024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335369600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017561.862035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.862038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017561.862040:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.862044:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.862046:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.862050:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017561.862054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017561.862062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335369728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017561.862071:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.862074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017561.862075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.862079:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.862081:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.862085:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017561.862089:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017561.862091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335369472:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017561.862099:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335369856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017561.862108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.862112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017561.862113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.862118:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.862120:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.862124:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017561.862128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017561.862136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335369984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017561.862144:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.862147:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017561.862149:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.862152:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.862155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.862159:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017561.862163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017561.862170:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335370112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017561.862176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017561.862178:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017561.862181:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017561.862182:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017561.862186:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1826 00000100:00080000:2.0:1734017561.862189:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017561.862192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017561.862197:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017561.862200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86dea00 x1818247335369472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017561.862205:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335370240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017561.862230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335369472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335369472:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017561.862256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335369600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017561.862274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017561.862281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86dce00 x1818247335369600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017561.862293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335369600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335369600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017561.862312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335369728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017561.862327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017561.862333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86de300 x1818247335369728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017561.862343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335369728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335369728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017561.862389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335369856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017561.862403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017561.862409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86df480 x1818247335369856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017561.862420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335369856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862429:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862432:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335369856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017561.862438:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017561.862450:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017561.862456:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86dd500 x1818247335369984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017561.862465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335369984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335369984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017561.862485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335370112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017561.862501:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017561.862507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86de680 x1818247335370112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017561.862518:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335370112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862530:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335370112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017561.862535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335370240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017561.862548:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017561.862554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017561/real 1734017561] req@ffff8800a86dd180 x1818247335370240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017561.862564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335370240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017561.862571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017561.862573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017561.862576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335370240:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017562.098002:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017562.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017563.099975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017563.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017564.101918:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017564.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017565.103953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017565.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017566.106001:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017566.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017566.869823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017566.869866:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017566.869871:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.869881:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.869890:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.869897:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017566.869931:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335370368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017566.869963:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017566.869967:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017566.869969:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.869974:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.869977:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.869981:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017566.869990:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017566.870000:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335370496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017566.870010:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017566.870013:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017566.870016:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.870019:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.870022:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.870025:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017566.870030:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017566.870038:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335370624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017566.870046:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017566.870049:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017566.870051:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.870054:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.870056:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.870060:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017566.870064:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017566.870074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335370752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017566.870082:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734017566.870083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335370368:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017566.870086:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017566.870087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.870091:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.870093:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.870097:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017566.870101:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017566.870109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335370880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017566.870117:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017566.870120:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017566.870122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.870125:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.870128:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.870131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017566.870135:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017566.870142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335371008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017566.870149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1734017566.870152:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017566.870152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017566.870154:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017566.870157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1831 00000100:00080000:2.0:1734017566.870160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017566.870163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017566.870167:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017566.870173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86df800 x1818247335370368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017566.870174:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335371136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017566.870201:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335370368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870211:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870215:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870219:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335370368:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017566.870226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335370496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017566.870243:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017566.870249:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86ddc00 x1818247335370496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017566.870260:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335370496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335370496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017566.870280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335370624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017566.870298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017566.870304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86dc000 x1818247335370624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017566.870314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335370624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870321:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335370624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017566.870354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335370752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017566.870367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017566.870373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86dd180 x1818247335370752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017566.870383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335370752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335370752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017566.870402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335370880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017566.870413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017566.870419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86de680 x1818247335370880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017566.870429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335370880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335370880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017566.870447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017566.870462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017566.870469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86dd500 x1818247335371008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017566.870479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335371008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017566.870496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017566.870508:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017566.870514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017566/real 1734017566] req@ffff8800a86df480 x1818247335371136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017566.870524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335371136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017566.870531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017566.870533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017566.870536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371136:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017567.107982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017567.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017568.110006:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017568.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017569.111939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017569.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017570.114011:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017570.213987:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017571.116012:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017571.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017571.877855:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017571.877864:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017571.877869:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.877879:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.877887:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.877894:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017571.877931:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335371264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017571.877963:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017571.877967:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017571.877969:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.877973:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.877976:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.877980:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017571.877989:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017571.878000:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335371392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017571.878010:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017571.878013:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017571.878015:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.878018:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.878020:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.878024:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017571.878028:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017571.878036:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335371520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017571.878052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017571.878055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017571.878056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.878060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.878062:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.878066:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017571.878070:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017571.878076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335371264:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017571.878079:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335371648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017571.878088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017571.878091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017571.878093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.878096:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.878098:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.878102:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017571.878106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017571.878113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335371776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017571.878121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017571.878124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017571.878125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.878129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.878141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.878145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:0.0:1734017571.878146:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017571.878150:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017571.878158:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335371904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017571.878166:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734017571.878167:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86de300 x1818247335371264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017571.878169:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017571.878170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017571.878173:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1836 00000100:00080000:2.0:1734017571.878176:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017571.878179:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017571.878183:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017571.878189:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335371264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017571.878191:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335372032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734017571.878200:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335371264:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017571.878216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335371392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017571.878232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017571.878239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86dce00 x1818247335371392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017571.878251:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335371392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017571.878259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335371392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017571.878270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335371520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017571.878283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017571.878289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86dea00 x1818247335371520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017571.878299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335371520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017571.878307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335371520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017571.878341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335371648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017571.878353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017571.878359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86dfb80 x1818247335371648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017571.878370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335371648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017571.878377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335371648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017571.878388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017571.878400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017571.878405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86ddf80 x1818247335371776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017571.878415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335371776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017571.878423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017571.878434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017571.878449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017571.878455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86dc380 x1818247335371904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017571.878466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335371904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017571.878473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335371904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017571.878483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017571.878495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017571.878501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017571/real 1734017571] req@ffff8800a86dca80 x1818247335372032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017571.878511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335372032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017571.878519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017571.878521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017571.878524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372032:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017572.118006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017572.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017573.120057:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017573.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017574.121930:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017574.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017575.123951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017575.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017576.125946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017576.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017576.885832:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.885840:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017576.885845:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.885854:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00080000:2.0:1734017576.885864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.885871:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017576.885907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335372160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017576.885940:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.885944:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017576.885947:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.885951:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00080000:2.0:1734017576.885955:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.885958:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017576.885968:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017576.885978:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335372288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017576.885988:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.885991:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017576.885993:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.885996:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00080000:2.0:1734017576.885999:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.886003:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017576.886007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017576.886015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335372416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017576.886023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.886026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017576.886027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.886030:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00080000:2.0:1734017576.886033:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.886036:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017576.886040:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017576.886043:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335372160:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017576.886049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335372544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017576.886059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.886062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017576.886063:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.886066:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00080000:2.0:1734017576.886069:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.886072:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017576.886076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017576.886083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335372672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017576.886091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.886094:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017576.886096:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.886099:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00080000:2.0:1734017576.886102:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.886105:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017576.886109:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017576.886112:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017576.886116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335372800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017576.886124:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017576.886127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017576.886129:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017576.886132:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1841 00000100:00100000:0.0:1734017576.886133:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86dd880 x1818247335372160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017576.886136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017576.886139:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017576.886143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017576.886150:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335372928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017576.886152:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335372160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886163:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335372160:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017576.886179:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335372288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017576.886195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017576.886202:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86df100 x1818247335372288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017576.886213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335372288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886221:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886227:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335372288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017576.886232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335372416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017576.886246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017576.886252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86dc700 x1818247335372416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017576.886262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335372416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335372416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017576.886303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335372544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017576.886315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017576.886321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86dca80 x1818247335372544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017576.886332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335372544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335372544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017576.886351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017576.886363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017576.886369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86dc380 x1818247335372672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017576.886379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335372672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017576.886399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017576.886412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017576.886418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86ddf80 x1818247335372800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017576.886428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335372800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886438:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017576.886446:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017576.886458:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017576.886464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017576/real 1734017576] req@ffff8800a86dfb80 x1818247335372928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017576.886474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335372928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017576.886481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017576.886483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017576.886487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335372928:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017577.127935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017577.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017578.129972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017578.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017579.131968:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017579.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017580.133940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017580.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017581.135989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017581.213942:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017581.893828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.893837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017581.893842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.893851:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00080000:2.0:1734017581.893860:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.893867:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017581.893903:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335373056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017581.893935:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.893940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017581.893941:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.893946:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00080000:2.0:1734017581.893949:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.893952:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017581.893962:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017581.893972:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335373184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017581.893981:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.893985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017581.893987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.893991:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00080000:2.0:1734017581.893994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.893997:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017581.894001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017581.894009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335373312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017581.894018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.894021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017581.894022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.894026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00080000:2.0:1734017581.894028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.894031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017581.894035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017581.894045:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335373440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017581.894051:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335373056:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017581.894053:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.894057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017581.894058:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.894062:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00080000:2.0:1734017581.894065:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.894068:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017581.894073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017581.894080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335373568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017581.894089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.894092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017581.894093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.894096:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00080000:2.0:1734017581.894099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.894102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017581.894106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017581.894112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335373696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017581.894117:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017581.894120:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017581.894123:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017581.894124:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017581.894127:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1846 00000100:00100000:0.0:1734017581.894139:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86dea00 x1818247335373056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017581.894140:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017581.894144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017581.894148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017581.894155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335373824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017581.894159:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335373056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894169:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894172:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894177:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335373056:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017581.894184:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335373184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017581.894200:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017581.894207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86dce00 x1818247335373184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017581.894218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335373184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894226:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335373184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017581.894237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335373312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017581.894250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017581.894256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86de300 x1818247335373312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017581.894274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335373312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335373312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017581.894313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335373440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017581.894325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017581.894331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86df480 x1818247335373440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017581.894341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335373440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335373440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017581.894359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335373568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017581.894371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017581.894377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86dd500 x1818247335373568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017581.894387:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335373568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335373568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017581.894406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335373696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017581.894418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017581.894423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86de680 x1818247335373696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017581.894433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335373696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894446:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335373696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017581.894451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335373824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017581.894462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017581.894468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017581/real 1734017581] req@ffff8800a86dd180 x1818247335373824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017581.894478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335373824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017581.894485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017581.894487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017581.894490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335373824:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017582.137953:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017582.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017583.139948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017583.213788:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017584.141949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017584.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017585.143976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017585.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017586.145978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017586.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017586.901840:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.901851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017586.901857:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.901870:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.901881:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.901891:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017586.901939:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335373952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017586.901975:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.901980:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017586.901982:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.901987:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.901991:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.901995:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017586.902006:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017586.902017:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335374080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017586.902027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.902030:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017586.902032:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.902037:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.902040:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.902044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017586.902050:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017586.902058:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335374208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017586.902068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.902071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017586.902072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.902076:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.902079:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.902082:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017586.902086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017586.902091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335373952:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017586.902096:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335374336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017586.902105:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.902109:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017586.902110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.902114:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.902116:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.902120:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017586.902124:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017586.902133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335374464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017586.902142:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.902145:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017586.902147:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.902151:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.902155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.902159:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017586.902163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017586.902170:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335374592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017586.902178:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017586.902181:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1734017586.902182:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017586.902182:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017586.902186:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1851 00000100:00080000:2.0:1734017586.902189:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017586.902192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017586.902196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017586.902204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335374720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017586.902209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86dc000 x1818247335373952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335373952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335373952:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017586.902257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017586.902275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017586.902282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86ddc00 x1818247335374080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335374080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017586.902313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017586.902328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017586.902334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86df800 x1818247335374208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335374208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017586.902390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017586.902403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017586.902410:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86dd180 x1818247335374336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335374336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017586.902439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335374464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017586.902451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017586.902457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86de680 x1818247335374464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335374464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335374464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017586.902486:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335374592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017586.902545:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017586.902554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86dd500 x1818247335374592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902565:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335374592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902576:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902579:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335374592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017586.902584:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335374720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017586.902598:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017586.902604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017586/real 1734017586] req@ffff8800a86df480 x1818247335374720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017586.902613:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335374720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017586.902621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017586.902623:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017586.902627:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335374720:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017587.147956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017587.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017588.149983:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017588.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017589.151970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017589.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017590.153980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017590.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017591.155979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017591.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017591.909850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.909859:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017591.909863:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017591.909872:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.909881:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.909889:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017591.909924:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335374848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017591.909957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.909961:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017591.909963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017591.909967:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.909970:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.909974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017591.909983:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017591.909993:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335374976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017591.910003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.910006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017591.910007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017591.910011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.910014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.910017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017591.910021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017591.910029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335375104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017591.910037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.910043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017591.910045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017591.910048:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.910051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.910054:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017591.910058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017591.910064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335374848:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017591.910068:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335375232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017591.910076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.910079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017591.910081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017591.910084:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.910087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.910090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017591.910094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017591.910101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335375360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017591.910109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.910112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017591.910113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017591.910117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.910120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.910123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017591.910128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734017591.910131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017591.910135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335375488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017591.910144:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017591.910146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017591.910148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734017591.910152:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86de300 x1818247335374848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017591.910152:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1856 00000100:00080000:2.0:1734017591.910154:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017591.910157:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017591.910161:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017591.910168:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335375616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017591.910172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335374848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910183:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910186:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910191:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335374848:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017591.910198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017591.910214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017591.910221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86dce00 x1818247335374976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017591.910232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335374976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335374976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017591.910251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335375104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017591.910264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017591.910270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86dea00 x1818247335375104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017591.910280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335375104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335375104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017591.910327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335375232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017591.910340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017591.910346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86dfb80 x1818247335375232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017591.910356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335375232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335375232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017591.910374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335375360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017591.910386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017591.910391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86ddf80 x1818247335375360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017591.910401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335375360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910459:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335375360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017591.910465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335375488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017591.910484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017591.910492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86dc380 x1818247335375488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017591.910502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335375488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335375488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017591.910520:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335375616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017591.910538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017591.910543:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017591/real 1734017591] req@ffff8800a86dca80 x1818247335375616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017591.910553:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335375616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017591.910560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017591.910562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017591.910565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335375616:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017592.157961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017592.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017593.159988:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017593.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017594.161934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017594.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017595.163926:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017595.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017596.165964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017596.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017596.917834:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.917843:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017596.917847:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.917857:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.917867:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017596.917875:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017596.917911:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335375744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017596.917952:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.917956:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017596.917958:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.917962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.917965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017596.917968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017596.917978:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017596.917988:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335375872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017596.917997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.918000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017596.918002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.918005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.918008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017596.918011:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017596.918015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017596.918023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335376000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017596.918031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.918034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017596.918036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.918039:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.918041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017596.918046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017596.918050:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017596.918060:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335375744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017596.918060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335376128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017596.918068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.918072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017596.918073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.918076:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.918079:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017596.918082:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017596.918086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017596.918094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335376256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017596.918102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.918105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017596.918106:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.918109:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.918112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017596.918115:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017596.918119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017596.918126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335376384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734017596.918127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017596.918134:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017596.918138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017596.918139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017596.918143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1861 00000100:00080000:2.0:1734017596.918146:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017596.918148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86dc700 x1818247335375744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017596.918149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017596.918153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017596.918160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335376512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017596.918176:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335375744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918186:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918189:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918194:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335375744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734017596.918201:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335375872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017596.918218:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017596.918224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86df100 x1818247335375872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017596.918235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335375872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335375872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017596.918254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017596.918267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017596.918273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86dd880 x1818247335376000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017596.918283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335376000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918297:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017596.918369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017596.918387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017596.918395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86dca80 x1818247335376128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017596.918406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335376128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017596.918424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335376256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734017596.918436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017596.918441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86dc380 x1818247335376256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017596.918451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335376256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335376256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017596.918470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335376384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017596.918484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017596.918490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86ddf80 x1818247335376384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017596.918501:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335376384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335376384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017596.918518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335376512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734017596.918530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734017596.918536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017596/real 1734017596] req@ffff8800a86dfb80 x1818247335376512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734017596.918546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335376512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734017596.918553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734017596.918555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734017596.918558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335376512:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017597.167963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017597.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017598.170112:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017598.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017599.171940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017599.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017600.173888:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017600.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017601.175838:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017601.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017601.925813:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.925822:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017601.925826:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.925836:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.925845:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017601.925852:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017601.925887:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335376640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017601.925920:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.925924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017601.925926:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.925930:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.925934:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017601.925938:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017601.925947:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017601.925958:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335376768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017601.925968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.925971:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017601.925973:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.925976:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.925979:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017601.925982:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017601.925986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017601.925994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335376896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017601.926003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.926006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017601.926007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.926011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.926013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017601.926017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017601.926020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734017601.926029:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335376640:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017601.926030:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335377024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017601.926045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.926048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017601.926050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.926053:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.926055:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017601.926059:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017601.926062:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017601.926070:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335377152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017601.926077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.926080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017601.926082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.926085:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.926088:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017601.926091:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017601.926095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017601.926102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335377280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017601.926109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017601.926112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017601.926113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017601.926117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1866 00000100:00080000:2.0:1734017601.926120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734017601.926121:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376768:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734017601.926123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017601.926128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017601.926135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335377408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017601.926142:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017601.926166:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017601.926181:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335377152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734017601.926195:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335377280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017601.926210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335377408:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017602.177834:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017602.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017603.179846:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017603.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017604.181826:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017604.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:0.0:1734017604.932060:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017604.932114:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017604.932133:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017604.932167:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017604.932175:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017604.932229:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017604.932237:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017604.932244:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017604.932251:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017604.932283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86df480 x1818247335377024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335377024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017604.932343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86dd500 x1818247335377152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335377152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335377152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017604.932375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86de680 x1818247335377280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335377280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335377280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017604.932405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86dd180 x1818247335377408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335377408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335377408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017604.932442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86dea00 x1818247335376640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335376640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335376640:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017604.932499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86dce00 x1818247335376768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335376768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932538:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017604.932547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017601/real 1734017604] req@ffff8800a86de300 x1818247335376896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017604.932564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335376896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017604.932587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017604.932590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017604.932595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335376896:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017605.183908:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017605.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017606.185941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017606.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017606.933847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.933856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017606.933861:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.933871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.933881:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.933889:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017606.933924:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335377536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017606.933958:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.933962:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017606.933964:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.933968:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.933971:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.933975:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017606.933985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017606.933996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335377664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017606.934006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.934009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017606.934011:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.934014:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.934017:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.934021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017606.934025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017606.934033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335377792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017606.934041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.934044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017606.934046:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.934049:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.934051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.934055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017606.934058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017606.934068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335377536:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017606.934068:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335377920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017606.934076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.934080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017606.934081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.934086:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.934088:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.934091:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017606.934095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017606.934103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335378048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017606.934111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.934114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017606.934115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.934118:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.934121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.934124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017606.934128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017606.934135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335378176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017606.934139:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017606.934143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017606.934145:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017606.934147:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017606.934184:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1871 00000100:00080000:2.0:1734017606.934187:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017606.934190:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017606.934195:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017606.934200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86df800 x1818247335377536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017606.934204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335378304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017606.934221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335377536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335377536:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017606.934247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017606.934267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017606.934274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86ddc00 x1818247335377664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017606.934285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335377664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017606.934305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017606.934320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017606.934327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86dc000 x1818247335377792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017606.934338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335377792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017606.934374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017606.934387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017606.934393:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86de300 x1818247335377920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017606.934403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335377920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335377920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017606.934422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017606.934433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017606.934439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86dce00 x1818247335378048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017606.934450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335378048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017606.934468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017606.934479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017606.934485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86dea00 x1818247335378176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017606.934494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335378176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017606.934512:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017606.934524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017606.934530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017606/real 1734017606] req@ffff8800a86dd180 x1818247335378304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017606.934540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335378304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017606.934547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017606.934550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017606.934553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378304:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017607.188014:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017607.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017608.190001:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017608.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017609.191941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017609.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017610.193908:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017610.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017611.195947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017611.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017611.941847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.941855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017611.941860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.941871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.941880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.941887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017611.941922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335378432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017611.941955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.941959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017611.941961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.941965:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.941968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.941972:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017611.941981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017611.941992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335378560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017611.942001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.942005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017611.942007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.942011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.942013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.942017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017611.942021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017611.942029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335378688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017611.942037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.942040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017611.942042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.942045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.942047:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.942051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017611.942055:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017611.942108:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335378432:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017611.942112:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335378816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017611.942123:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.942126:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017611.942128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.942131:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.942134:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.942137:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017611.942142:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017611.942152:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335378944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017611.942161:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.942164:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017611.942165:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.942169:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.942171:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.942174:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017611.942177:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017611.942178:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017611.942186:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335379072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017611.942194:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017611.942196:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1734017611.942198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86de680 x1818247335378432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017611.942198:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017611.942202:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1876 00000100:00080000:2.0:1734017611.942204:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017611.942207:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017611.942211:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017611.942218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335378432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017611.942218:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335379200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017611.942228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942236:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335378432:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017611.942244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335378560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017611.942261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017611.942268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86dd500 x1818247335378560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017611.942279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335378560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017611.942286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335378560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017611.942298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335378688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017611.942314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017611.942320:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86df480 x1818247335378688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017611.942331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335378688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017611.942339:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335378688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017611.942367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335378816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017611.942379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017611.942385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86dfb80 x1818247335378816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017611.942395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335378816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017611.942402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335378816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017611.942414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017611.942426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017611.942432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86ddf80 x1818247335378944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017611.942449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335378944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017611.942456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335378944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017611.942467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017611.942478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017611.942484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86dc380 x1818247335379072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017611.942494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335379072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017611.942501:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017611.942511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017611.942523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017611.942529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017611/real 1734017611] req@ffff8800a86dca80 x1818247335379200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017611.942538:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335379200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017611.942546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017611.942549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017611.942552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379200:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017612.197922:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017612.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017613.199995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017613.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017614.201967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017614.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017615.203970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017615.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017616.205973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017616.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017616.949839:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017616.949848:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017616.949852:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.949862:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.949871:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.949878:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017616.949913:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335379328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017616.949947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017616.949951:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017616.949953:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.949957:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.949961:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.949964:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017616.949974:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017616.950011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335379456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017616.950021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017616.950024:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017616.950026:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.950030:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.950033:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.950036:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017616.950041:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017616.950049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335379584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017616.950058:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017616.950062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017616.950063:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.950067:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.950069:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.950073:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950077:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335379328:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734017616.950077:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017616.950087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335379712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017616.950096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017616.950099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017616.950100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.950104:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.950106:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.950110:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017616.950114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017616.950122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335379840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017616.950130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017616.950132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017616.950134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.950138:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.950141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.950145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017616.950146:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017616.950149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017616.950156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335379968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017616.950164:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017616.950167:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86dd880 x1818247335379328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017616.950167:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017616.950168:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017616.950172:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1881 00000100:00080000:2.0:1734017616.950174:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017616.950177:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017616.950181:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017616.950186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335379328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017616.950189:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335380096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017616.950197:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950200:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950205:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335379328:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017616.950212:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335379456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017616.950229:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86df100 x1818247335379456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017616.950247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335379456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017616.950255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335379456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017616.950267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335379584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017616.950283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86dc700 x1818247335379584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017616.950299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335379584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017616.950307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335379584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017616.950341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335379712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017616.950353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86dca80 x1818247335379712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017616.950369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335379712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017616.950377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335379712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017616.950388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017616.950399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86dc380 x1818247335379840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017616.950422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335379840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017616.950430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017616.950441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017616.950452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86ddf80 x1818247335379968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017616.950468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335379968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017616.950475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335379968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017616.950485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017616.950497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017616.950503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017616/real 1734017616] req@ffff8800a86dfb80 x1818247335380096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017616.950513:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335380096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017616.950520:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017616.950523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017616.950527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380096:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017617.207921:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017617.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017618.210038:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017618.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017619.212014:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017619.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734017620.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017620.213916:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017621.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017621.215959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017621.957828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.957837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017621.957842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.957851:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00080000:2.0:1734017621.957861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.957869:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017621.957936:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335380224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017621.957969:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.957973:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017621.957975:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.957979:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00080000:2.0:1734017621.957983:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.957986:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017621.957996:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017621.958007:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335380352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017621.958017:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.958020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017621.958022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.958025:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00080000:2.0:1734017621.958028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.958031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017621.958035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017621.958044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335380480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017621.958052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.958055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017621.958056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.958060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00080000:2.0:1734017621.958063:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.958066:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017621.958070:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017621.958080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335380608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017621.958083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335380224:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017621.958089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.958092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017621.958093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.958097:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00080000:2.0:1734017621.958099:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.958103:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017621.958107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017621.958115:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335380736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017621.958123:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.958126:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017621.958127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.958131:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00080000:2.0:1734017621.958133:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.958136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017621.958140:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017621.958147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335380864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017621.958154:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017621.958156:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017621.958159:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017621.958160:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017621.958164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1886 00000100:00100000:1.0:1734017621.958175:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86df480 x1818247335380224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017621.958176:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017621.958179:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017621.958183:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017621.958191:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335380992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017621.958195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335380224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958206:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958209:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958214:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335380224:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017621.958221:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335380352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017621.958237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017621.958243:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86dd500 x1818247335380352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017621.958254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335380352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335380352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017621.958274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335380480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017621.958290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017621.958297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86de680 x1818247335380480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017621.958307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335380480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958317:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335380480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017621.958345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335380608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017621.958357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017621.958363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86dd180 x1818247335380608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017621.958372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335380608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335380608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017621.958391:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017621.958402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017621.958408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86dea00 x1818247335380736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017621.958425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335380736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958433:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958435:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958438:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017621.958443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017621.958454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017621.958460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86dce00 x1818247335380864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017621.958470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335380864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017621.958487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017621.958499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017621.958504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017621/real 1734017621] req@ffff8800a86de300 x1818247335380992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017621.958514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335380992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017621.958521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017621.958524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017621.958527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335380992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017622.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017622.217957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017623.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017623.219944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017624.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017624.221957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017625.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017625.223937:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017626.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017626.225990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017626.965803:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.965852:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017626.965856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.965866:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.965875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017626.965882:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017626.965918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335381120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017626.965950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.965955:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017626.965957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.965962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.965965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017626.965969:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017626.965979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017626.965990:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335381248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017626.965999:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.966003:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017626.966005:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.966009:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.966011:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017626.966014:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017626.966019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017626.966027:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335381376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017626.966035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.966038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017626.966039:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.966043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.966045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017626.966049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017626.966052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017626.966062:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335381504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017626.966064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335381120:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017626.966072:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.966076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017626.966077:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.966081:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.966083:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017626.966086:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017626.966090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017626.966097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335381632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017626.966105:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.966108:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017626.966110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.966113:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.966116:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017626.966119:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017626.966123:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017626.966130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335381760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017626.966132:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017626.966137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017626.966140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017626.966142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017626.966146:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1891 00000100:00080000:2.0:1734017626.966149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017626.966153:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86dc000 x1818247335381120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017626.966153:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017626.966157:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017626.966164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335381888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017626.966172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335381120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966183:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966186:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966191:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335381120:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017626.966198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335381248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017626.966215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017626.966221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86ddc00 x1818247335381248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017626.966232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335381248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335381248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017626.966251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335381376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017626.966268:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017626.966274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86df800 x1818247335381376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017626.966285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335381376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966298:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335381376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017626.966321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335381504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017626.966334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017626.966340:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86de300 x1818247335381504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017626.966350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335381504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335381504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017626.966368:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335381632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017626.966380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017626.966385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86dce00 x1818247335381632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017626.966402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335381632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335381632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017626.966421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335381760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017626.966432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017626.966438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86dea00 x1818247335381760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017626.966448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335381760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335381760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017626.966465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335381888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017626.966477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017626.966483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017626/real 1734017626] req@ffff8800a86dd180 x1818247335381888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017626.966493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335381888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017626.966500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017626.966503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017626.966506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335381888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017627.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017627.227981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017628.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017628.229998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017629.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017629.231966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017630.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017630.233978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017631.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017631.235976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017631.973830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.973838:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017631.973843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.973853:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00080000:2.0:1734017631.973862:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.973870:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017631.973905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335382016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017631.973937:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.973942:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017631.973943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.973948:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00080000:2.0:1734017631.973951:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.973955:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017631.973964:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017631.973975:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335382144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017631.973985:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.973988:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017631.973990:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.973994:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00080000:2.0:1734017631.973996:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.973999:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017631.974004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017631.974011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335382272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017631.974020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.974022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017631.974024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.974027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00080000:2.0:1734017631.974029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.974033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017631.974037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017631.974045:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335382016:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017631.974046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335382400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017631.974055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.974058:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017631.974059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.974064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00080000:2.0:1734017631.974066:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.974069:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017631.974073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017631.974081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335382528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017631.974089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.974092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017631.974094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.974097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00080000:2.0:1734017631.974102:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.974105:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017631.974109:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017631.974111:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017631.974116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335382656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017631.974124:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017631.974127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017631.974128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017631.974132:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1896 00000100:00100000:1.0:1734017631.974133:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86de680 x1818247335382016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017631.974135:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017631.974138:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017631.974143:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017631.974150:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335382784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017631.974153:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335382016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974163:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974171:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335382016:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017631.974178:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335382144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017631.974194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017631.974201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86dd500 x1818247335382144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017631.974212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335382144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974222:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335382144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017631.974231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335382272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017631.974253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017631.974259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86df480 x1818247335382272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017631.974270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335382272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974277:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335382272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017631.974308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335382400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017631.974320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017631.974326:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86dfb80 x1818247335382400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017631.974336:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335382400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335382400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017631.974354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335382528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017631.974366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017631.974371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86ddf80 x1818247335382528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017631.974388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335382528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335382528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017631.974406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335382656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017631.974417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017631.974423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86dc380 x1818247335382656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017631.974433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335382656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335382656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017631.974450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335382784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017631.974462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017631.974468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017631/real 1734017631] req@ffff8800a86dca80 x1818247335382784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017631.974478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335382784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017631.974485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017631.974488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017631.974491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335382784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017632.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017632.238000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017633.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017633.239965:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017634.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017634.241974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017635.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017635.243951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017636.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017636.245958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017636.981823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017636.981831:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017636.981836:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.981845:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.981854:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017636.981861:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017636.981896:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335382912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017636.981932:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017636.981937:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017636.981939:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.981943:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.981946:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017636.981950:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017636.981959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017636.981970:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335383040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017636.981979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017636.981982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017636.981985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.981989:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.981991:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017636.981994:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017636.981999:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017636.982007:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335383168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017636.982012:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335382912:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017636.982016:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017636.982019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017636.982020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.982024:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.982026:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017636.982030:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017636.982034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017636.982043:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335383296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017636.982050:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017636.982054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017636.982055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.982058:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.982061:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017636.982064:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017636.982069:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017636.982076:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335383424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017636.982083:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017636.982085:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017636.982089:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017636.982091:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.982094:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.982099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017636.982102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982104:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86dc700 x1818247335382912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017636.982106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017636.982113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335383552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017636.982121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017636.982123:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335382912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017636.982124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017636.982125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017636.982129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1901 00000100:00080000:2.0:1734017636.982131:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734017636.982134:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017636.982134:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:1.0:1734017636.982137:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734017636.982138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017636.982142:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335382912:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017636.982146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335383680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017636.982150:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017636.982166:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86df100 x1818247335383040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017636.982184:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335383040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017636.982192:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017636.982195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017636.982198:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017636.982220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017636.982237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982243:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86dd880 x1818247335383168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017636.982254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335383168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017636.982262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017636.982264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017636.982267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017636.982272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017636.982283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86dca80 x1818247335383296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017636.982299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335383296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017636.982306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017636.982308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017636.982311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017636.982317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335383424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017636.982329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982335:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86dc380 x1818247335383424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017636.982353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335383424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017636.982360:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017636.982363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017636.982366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335383424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017636.982371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335383552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017636.982383:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982388:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86ddf80 x1818247335383552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017636.982398:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335383552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017636.982405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017636.982408:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017636.982411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335383552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017636.982416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335383680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017636.982428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017636.982434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017636/real 1734017636] req@ffff8800a86dfb80 x1818247335383680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017636.982443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335383680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017636.982451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017636.982454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017636.982457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335383680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017637.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017637.247974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017638.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017638.249941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017639.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017639.251969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017640.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017640.253937:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017641.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017641.255937:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017641.989825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017641.989834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017641.989839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.989849:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.989858:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.989866:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017641.989902:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335383808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017641.989936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017641.989940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017641.989942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.989946:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.989950:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.989953:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017641.989963:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017641.989974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335383936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017641.989984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017641.989987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017641.989989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.989993:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.989995:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.989998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017641.990003:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017641.990011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335384064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017641.990020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017641.990023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017641.990025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.990028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.990031:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.990034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017641.990038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017641.990046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335383808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017641.990047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335384192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017641.990056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017641.990059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017641.990061:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.990064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.990066:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.990070:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017641.990074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017641.990081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335384320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017641.990089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017641.990092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017641.990094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.990097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.990110:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.990114:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017641.990115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017641.990119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017641.990126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335384448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017641.990134:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017641.990136:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86df480 x1818247335383808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017641.990137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017641.990139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017641.990142:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1906 00000100:00080000:2.0:1734017641.990145:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017641.990148:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017641.990152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017641.990156:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335383808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017641.990159:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335384576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017641.990166:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990169:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990174:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335383808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017641.990182:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017641.990198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017641.990205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86dd500 x1818247335383936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017641.990216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335383936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017641.990224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990230:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335383936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017641.990235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017641.990252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017641.990258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86de680 x1818247335384064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017641.990269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335384064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017641.990277:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017641.990306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017641.990318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017641.990324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86dd180 x1818247335384192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017641.990334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335384192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017641.990341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017641.990352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335384320:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017641.990363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017641.990369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86dea00 x1818247335384320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017641.990385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335384320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017641.990393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335384320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017641.990403:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335384448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017641.990414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017641.990420:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86dce00 x1818247335384448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017641.990430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335384448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017641.990437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335384448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017641.990447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335384576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017641.990459:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017641.990465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017641/real 1734017641] req@ffff8800a86de300 x1818247335384576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017641.990475:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335384576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017641.990482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017641.990485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017641.990488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335384576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017642.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017642.257980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017643.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017643.259919:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017644.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017644.261951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017645.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017645.263981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017646.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017646.265946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017646.997842:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.997852:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017646.997856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017646.997866:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.997875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.997882:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017646.997918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335384704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017646.997951:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.997955:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017646.997957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017646.997961:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.997965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.997968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017646.997978:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017646.997988:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335384832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017646.997998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.998001:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017646.998003:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017646.998006:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.998009:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.998012:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017646.998017:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017646.998024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335384960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017646.998033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.998036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017646.998037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017646.998042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.998044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.998048:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017646.998052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017646.998058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335384704:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017646.998061:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335385088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017646.998070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.998073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017646.998075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017646.998078:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.998080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.998084:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017646.998088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017646.998095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335385216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017646.998103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.998106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017646.998108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017646.998111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.998115:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.998119:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017646.998123:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017646.998126:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017646.998130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335385344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017646.998140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017646.998143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017646.998144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017646.998147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86df800 x1818247335384704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017646.998148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1911 00000100:00080000:2.0:1734017646.998150:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017646.998154:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017646.998158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017646.998165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335385472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017646.998166:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335384704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998176:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998180:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335384704:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017646.998192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017646.998209:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017646.998216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86ddc00 x1818247335384832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017646.998227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335384832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017646.998246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017646.998263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017646.998269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86dc000 x1818247335384960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017646.998280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335384960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335384960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017646.998316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017646.998328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017646.998334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86de300 x1818247335385088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017646.998344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335385088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017646.998362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335385216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017646.998374:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017646.998380:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86dce00 x1818247335385216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017646.998397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335385216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335385216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017646.998414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335385344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017646.998425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017646.998431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86dea00 x1818247335385344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017646.998441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335385344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335385344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017646.998459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335385472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017646.998516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017646.998524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017646/real 1734017646] req@ffff8800a86dd180 x1818247335385472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017646.998535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335385472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017646.998543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017646.998546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017646.998549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335385472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017647.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017647.267974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017648.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017648.269952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017649.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017649.271964:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017650.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017650.273952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017651.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017651.275926:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017652.005831:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.005840:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017652.005845:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.005855:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00080000:2.0:1734017652.005864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.005872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017652.005908:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335385600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017652.005942:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.005946:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017652.005949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.005954:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00080000:2.0:1734017652.005957:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.005961:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017652.005970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017652.005981:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335385728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017652.005990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.005994:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017652.005996:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.005999:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00080000:2.0:1734017652.006002:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.006006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017652.006010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017652.006017:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335385856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017652.006026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.006029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017652.006030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.006033:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00080000:2.0:1734017652.006036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.006039:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017652.006043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017652.006051:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335385600:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017652.006052:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335385984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017652.006063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.006066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017652.006068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.006071:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00080000:2.0:1734017652.006073:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.006077:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017652.006081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017652.006088:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335386112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017652.006096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.006099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017652.006100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.006104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00080000:2.0:1734017652.006107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.006110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017652.006114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017652.006118:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017652.006122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335386240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017652.006130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017652.006133:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017652.006134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017652.006138:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1916 00000100:00100000:1.0:1734017652.006139:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86de680 x1818247335385600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017652.006141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017652.006145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017652.006149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017652.006156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335386368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017652.006159:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335385600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006173:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006178:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335385600:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017652.006185:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017652.006201:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017652.006208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86dd500 x1818247335385728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017652.006219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335385728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017652.006238:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017652.006259:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017652.006265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86df480 x1818247335385856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017652.006276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335385856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017652.006313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017652.006325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017652.006331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86dfb80 x1818247335385984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017652.006341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335385984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335385984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017652.006359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335386112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017652.006371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017652.006376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86ddf80 x1818247335386112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017652.006440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335386112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335386112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017652.006459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335386240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017652.006474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017652.006481:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86dc380 x1818247335386240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017652.006491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335386240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335386240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017652.006509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335386368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017652.006521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017652.006527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017652/real 1734017652] req@ffff8800a86dca80 x1818247335386368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017652.006536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335386368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017652.006544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017652.006547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017652.006550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335386368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017652.213787:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017652.277977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017653.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017653.279957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017654.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017654.281961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017655.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017655.283932:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017656.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017656.285983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017657.013834:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.013842:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017657.013846:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.013856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00080000:2.0:1734017657.013864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.013871:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017657.013904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335386496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017657.013936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.013940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017657.013942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.013946:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00080000:2.0:1734017657.013949:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.013953:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017657.013962:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017657.013973:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335386624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017657.013984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.013987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017657.013989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.013993:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00080000:2.0:1734017657.013996:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.013999:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017657.014003:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017657.014011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335386752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017657.014019:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.014022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017657.014024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.014027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00080000:2.0:1734017657.014030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.014033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017657.014037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017657.014042:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335386496:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017657.014046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335386880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017657.014055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.014058:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017657.014060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.014064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00080000:2.0:1734017657.014067:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.014070:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017657.014074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017657.014082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335387008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017657.014089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.014092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017657.014094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.014097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00080000:2.0:1734017657.014100:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.014103:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017657.014107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017657.014110:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017657.014114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335387136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017657.014122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017657.014125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017657.014126:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017657.014130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1921 00000100:00100000:1.0:1734017657.014131:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86dd880 x1818247335386496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017657.014132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017657.014135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017657.014139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017657.014146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335387264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017657.014151:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335386496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014161:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014164:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014169:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335386496:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017657.014176:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335386624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017657.014193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017657.014200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86df100 x1818247335386624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017657.014211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335386624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014219:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014224:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335386624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017657.014230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335386752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017657.014251:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017657.014257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86dc700 x1818247335386752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017657.014268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335386752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335386752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017657.014352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335386880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017657.014371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017657.014378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86dca80 x1818247335386880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017657.014389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335386880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335386880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017657.014407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017657.014419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017657.014425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86dc380 x1818247335387008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017657.014442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335387008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017657.014460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017657.014471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017657.014477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86ddf80 x1818247335387136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017657.014487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335387136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017657.014504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017657.014516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017657.014522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017657/real 1734017657] req@ffff8800a86dfb80 x1818247335387264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017657.014532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335387264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017657.014540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017657.014543:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017657.014546:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017657.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017657.287961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017658.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017658.289958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017659.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017659.291959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017660.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017660.293947:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017661.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017661.295993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017662.021823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.021832:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017662.021837:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.021847:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00080000:2.0:1734017662.021855:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.021863:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017662.021898:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335387392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017662.021931:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.021936:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017662.021938:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.021942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00080000:2.0:1734017662.021945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.021949:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017662.021959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017662.021969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335387520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017662.021979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.021982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017662.021985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.021989:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00080000:2.0:1734017662.021992:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.021995:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017662.022000:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017662.022008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335387648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017662.022016:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.022019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017662.022020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.022024:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00080000:2.0:1734017662.022026:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.022029:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017662.022033:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017662.022039:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335387392:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017662.022042:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335387776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017662.022051:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.022054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017662.022056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.022059:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00080000:2.0:1734017662.022061:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.022065:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017662.022069:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017662.022077:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335387904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017662.022085:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.022089:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017662.022090:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.022093:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00080000:2.0:1734017662.022096:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.022099:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017662.022103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017662.022106:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017662.022111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335388032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017662.022118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017662.022121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017662.022123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017662.022126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1926 00000100:00100000:1.0:1734017662.022127:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86df480 x1818247335387392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017662.022129:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017662.022132:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017662.022136:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017662.022143:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335388160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017662.022147:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335387392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022158:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022161:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022166:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335387392:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017662.022173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335387520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017662.022190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017662.022197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86dd500 x1818247335387520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017662.022209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335387520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022266:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335387520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017662.022276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335387648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017662.022298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017662.022305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86de680 x1818247335387648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017662.022316:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335387648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335387648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017662.022354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335387776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017662.022366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017662.022372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86dd180 x1818247335387776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017662.022382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335387776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022395:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335387776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017662.022400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017662.022412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017662.022418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86dea00 x1818247335387904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017662.022434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335387904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335387904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017662.022452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017662.022463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017662.022468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86dce00 x1818247335388032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017662.022478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335388032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017662.022495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017662.022506:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017662.022512:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017662/real 1734017662] req@ffff8800a86de300 x1818247335388160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017662.022522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335388160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017662.022529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017662.022532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017662.022535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017662.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017662.297951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017663.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017663.299962:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017664.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017664.301973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017665.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017665.303987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017666.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017666.305971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017667.029836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.029849:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017667.029855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.029869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00080000:2.0:1734017667.029880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.029890:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017667.029938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335388288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017667.029979:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.029984:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017667.029986:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.029991:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00080000:2.0:1734017667.029994:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.029997:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017667.030009:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017667.030021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335388416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017667.030030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.030034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017667.030036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.030040:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00080000:2.0:1734017667.030042:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.030046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017667.030051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017667.030060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335388544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017667.030070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.030073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017667.030075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.030079:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00080000:2.0:1734017667.030081:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.030084:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017667.030088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017667.030102:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335388288:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017667.030102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335388672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017667.030111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.030115:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017667.030116:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.030121:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00080000:2.0:1734017667.030123:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.030155:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017667.030161:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017667.030171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335388800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017667.030180:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.030183:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017667.030185:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.030189:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00080000:2.0:1734017667.030194:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.030197:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017667.030201:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017667.030209:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335388928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017667.030217:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017667.030220:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017667.030221:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017667.030225:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1931 00000100:00100000:1.0:1734017667.030226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335388416:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734017667.030228:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017667.030231:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017667.030236:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017667.030244:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335389056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017667.030250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335388544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017667.030276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335388672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017667.030294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017667.030309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017667.030324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017667.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017667.307967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017668.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017668.309993:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017669.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017669.311982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1734017670.036138:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017670.036193:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017670.036212:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017670.036248:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017670.036255:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017670.036262:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017670.036269:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017670.036276:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017670.036283:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017670.036358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86de300 x1818247335388672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335388672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.036400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.036405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.036410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335388672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017670.036418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86dce00 x1818247335388800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335388800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.036445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.036449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.036455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017670.036465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86dea00 x1818247335388928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335388928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.036495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.036499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.036504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335388928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017670.036514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86dd180 x1818247335389056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335389056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.036544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.036547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.036552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017670.036562:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86dc000 x1818247335388288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036582:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335388288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.036604:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.036608:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.036613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335388288:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017670.036623:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86ddc00 x1818247335388416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036640:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335388416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.036652:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.036656:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.036661:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335388416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017670.036671:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017667/real 1734017670] req@ffff8800a86df800 x1818247335388544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017670.036688:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335388544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017670.037167:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017670.037170:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017670.037175:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335388544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017670.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017670.313910:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017671.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017671.315959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017672.037808:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.037817:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017672.037822:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017672.037832:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.037840:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.037848:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017672.037886:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335389184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017672.037920:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.037924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017672.037926:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017672.037931:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.037934:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.037937:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017672.037947:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017672.037958:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335389312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017672.037968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.037971:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017672.037972:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017672.037976:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.037979:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.037982:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017672.037986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017672.037994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335389440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017672.038002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.038005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017672.038007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017672.038011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.038014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.038017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017672.038021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017672.038027:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335389184:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017672.038031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335389568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017672.038079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.038083:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017672.038085:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017672.038089:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.038091:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.038095:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017672.038100:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017672.038109:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335389696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017672.038119:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.038122:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017672.038123:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017672.038127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017672.038127:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.038130:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.038133:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017672.038137:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017672.038144:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335389824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017672.038148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86de680 x1818247335389184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017672.038153:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017672.038156:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017672.038158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017672.038161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1936 00000100:00080000:2.0:1734017672.038164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017672.038167:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017672.038171:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017672.038177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335389184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017672.038178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335389952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017672.038187:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038195:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335389184:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017672.038203:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335389312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017672.038220:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017672.038227:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86dd500 x1818247335389312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017672.038238:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335389312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017672.038245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335389312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017672.038256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335389440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017672.038274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017672.038280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86df480 x1818247335389440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017672.038290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335389440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017672.038297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038304:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335389440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017672.038326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335389568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017672.038339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017672.038345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86dfb80 x1818247335389568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017672.038356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335389568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017672.038363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335389568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017672.038374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017672.038385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017672.038391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86ddf80 x1818247335389696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017672.038400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335389696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017672.038408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038410:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017672.038419:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017672.038434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017672.038440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86dc380 x1818247335389824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017672.038451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335389824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017672.038458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017672.038470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017672.038481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017672.038487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017672/real 1734017672] req@ffff8800a86dca80 x1818247335389952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017672.038497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335389952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017672.038504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017672.038506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017672.038510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335389952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017672.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017672.317928:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017673.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017673.319945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017674.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017674.322057:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017675.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017675.324060:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017676.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017676.326010:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017677.045836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.045847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017677.045853:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.045865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.045877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.045886:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017677.045932:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335390080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017677.046014:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.046019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017677.046022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.046028:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.046032:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.046035:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017677.046047:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017677.046060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335390208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017677.046071:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.046075:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017677.046077:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.046081:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.046083:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.046087:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017677.046092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017677.046100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335390336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017677.046109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.046112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017677.046113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.046117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.046120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.046123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017677.046128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017677.046140:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335390080:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017677.046140:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335390464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017677.046151:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.046155:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017677.046157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.046160:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.046163:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.046167:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017677.046171:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017677.046180:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335390592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017677.046187:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.046190:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017677.046192:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.046196:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.046199:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.046203:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017677.046207:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017677.046215:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335390720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017677.046216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017677.046223:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017677.046226:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017677.046228:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017677.046232:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1941 00000100:00080000:2.0:1734017677.046235:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017677.046239:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86dc700 x1818247335390080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017677.046243:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017677.046252:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335390848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017677.046268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335390080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046285:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335390080:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017677.046298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335390208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017677.046317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86df100 x1818247335390208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017677.046335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335390208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335390208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017677.046355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335390336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017677.046370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86dd880 x1818247335390336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017677.046386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335390336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335390336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017677.046426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335390464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017677.046440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86dca80 x1818247335390464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017677.046457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335390464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335390464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017677.046475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335390592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017677.046487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86dc380 x1818247335390592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017677.046503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335390592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335390592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017677.046522:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335390720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017677.046538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86ddf80 x1818247335390720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017677.046555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335390720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335390720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017677.046573:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335390848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017677.046585:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017677.046591:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017677/real 1734017677] req@ffff8800a86dfb80 x1818247335390848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017677.046601:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335390848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017677.046608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017677.046610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017677.046613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335390848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017677.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017677.328031:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017678.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017678.330023:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017679.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017679.331957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017680.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017680.334002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017681.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017681.335984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017682.053804:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017682.053813:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017682.053817:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017682.053827:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.053838:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.053846:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017682.053912:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335390976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017682.053944:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017682.053949:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017682.053951:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017682.053955:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.053959:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.053963:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017682.053972:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017682.053984:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335391104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017682.053994:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017682.053998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017682.053999:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017682.054003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.054006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.054009:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017682.054013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017682.054021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335391232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017682.054029:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017682.054033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017682.054034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017682.054037:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.054040:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.054043:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017682.054047:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017682.054057:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335391360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017682.054065:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017682.054067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335390976:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017682.054068:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017682.054070:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017682.054074:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.054077:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.054080:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017682.054084:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017682.054092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335391488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017682.054099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017682.054102:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017682.054104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017682.054107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.054109:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.054112:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017682.054116:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017682.054123:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335391616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017682.054131:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017682.054134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017682.054135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017682.054136:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017682.054139:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1946 00000100:00080000:2.0:1734017682.054141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017682.054144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017682.054148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017682.054155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335391744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017682.054158:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86df480 x1818247335390976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054185:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335390976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054195:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054202:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335390976:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017682.054209:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335391104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017682.054226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017682.054232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86dd500 x1818247335391104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335391104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054251:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335391104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017682.054263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335391232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017682.054281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017682.054287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86de680 x1818247335391232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335391232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335391232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017682.054335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335391360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017682.054348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017682.054354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86df800 x1818247335391360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335391360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335391360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017682.054383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335391488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017682.054394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017682.054400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86ddc00 x1818247335391488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335391488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054417:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335391488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017682.054428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335391616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017682.054443:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017682.054449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86dc000 x1818247335391616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335391616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054466:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335391616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017682.054476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335391744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017682.054487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017682.054493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017682/real 1734017682] req@ffff8800a86dd180 x1818247335391744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017682.054503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335391744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017682.054511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017682.054513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017682.054516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335391744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017682.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017682.337975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017683.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017683.339950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017684.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017684.341974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017685.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017685.343983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017686.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017686.346026:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017687.061879:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.061891:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017687.061897:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.061909:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.061920:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017687.061929:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017687.061977:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335391872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017687.062015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.062020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017687.062022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.062027:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.062030:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017687.062034:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017687.062045:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017687.062057:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335392000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017687.062067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.062070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017687.062073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.062077:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.062080:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017687.062084:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017687.062089:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017687.062097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335392128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017687.062105:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.062109:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017687.062110:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.062114:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.062117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017687.062120:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017687.062124:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017687.062135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335392256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017687.062144:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.062148:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017687.062149:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.062154:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.062156:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017687.062159:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335391872:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017687.062160:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017687.062165:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017687.062173:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335392384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017687.062184:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.062187:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017687.062189:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.062192:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.062195:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017687.062198:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017687.062203:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017687.062210:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335392512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017687.062218:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017687.062221:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017687.062222:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017687.062226:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1951 00000100:00080000:2.0:1734017687.062229:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017687.062233:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017687.062234:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017687.062237:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017687.062245:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335392640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017687.062257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86dea00 x1818247335391872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335391872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335391872:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017687.062313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017687.062331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017687.062338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86dce00 x1818247335392000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335392000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017687.062369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017687.062382:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017687.062388:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86de300 x1818247335392128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062398:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335392128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062409:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062412:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017687.062439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017687.062452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017687.062458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86dd180 x1818247335392256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335392256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017687.062488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335392384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017687.062500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017687.062506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86dc000 x1818247335392384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335392384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335392384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017687.062535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335392512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017687.062550:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017687.062556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86ddc00 x1818247335392512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335392512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062576:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062579:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335392512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017687.062584:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335392640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017687.062596:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017687.062602:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017687/real 1734017687] req@ffff8800a86df800 x1818247335392640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017687.062612:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335392640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017687.062619:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017687.062621:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017687.062624:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335392640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017687.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017687.347978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017688.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017688.349973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017689.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017689.351970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017690.213896:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017690.353988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017691.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017691.355990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017692.069837:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.069847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017692.069855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.069867:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.069878:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.069886:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017692.069931:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335392768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017692.069970:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.069975:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017692.069977:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.069981:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.069985:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.069989:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017692.070000:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017692.070012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335392896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017692.070022:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.070025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017692.070027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.070031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.070034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.070038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017692.070043:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017692.070052:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335393024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017692.070068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.070072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017692.070073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.070077:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.070080:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.070083:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017692.070087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017692.070098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335392768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017692.070099:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335393152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017692.070108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.070112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017692.070113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.070117:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.070119:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.070123:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017692.070127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017692.070135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335393280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017692.070143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.070147:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017692.070148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.070152:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.070156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.070160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017692.070164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017692.070172:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335393408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017692.070175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017692.070180:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017692.070183:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017692.070184:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017692.070188:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1956 00000100:00080000:2.0:1734017692.070192:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017692.070196:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86de680 x1818247335392768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017692.070200:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017692.070208:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335393536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017692.070227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335392768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335392768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017692.070255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017692.070273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86dd500 x1818247335392896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017692.070291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335392896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070299:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335392896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017692.070311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017692.070325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86df480 x1818247335393024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017692.070341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335393024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070349:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017692.070380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017692.070393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86dfb80 x1818247335393152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017692.070410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335393152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070417:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017692.070429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335393280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017692.070441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86ddf80 x1818247335393280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017692.070457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335393280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335393280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017692.070476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335393408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017692.070491:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070498:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86dc380 x1818247335393408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017692.070508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335393408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335393408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017692.070526:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335393536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017692.070538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017692.070544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017692/real 1734017692] req@ffff8800a86dca80 x1818247335393536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017692.070554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335393536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017692.070561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017692.070563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017692.070566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335393536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734017692.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017692.357955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017693.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017693.359957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017694.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017694.361924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017695.213949:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017695.363972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017696.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017696.366006:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017697.077826:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.077837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017697.077842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.077854:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.077865:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.077874:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017697.077919:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335393664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017697.077966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.077970:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017697.077972:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.077977:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.077980:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.077984:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017697.077995:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017697.078006:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335393792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017697.078015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.078019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017697.078020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.078024:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.078027:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.078031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017697.078036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017697.078044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335393920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017697.078052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.078055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017697.078057:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.078060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.078063:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.078068:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017697.078072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017697.078084:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335393664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017697.078084:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335394048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017697.078093:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.078096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017697.078098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.078102:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.078104:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.078108:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017697.078113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017697.078121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335394176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017697.078129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.078132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017697.078134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.078137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.078141:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.078145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017697.078149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017697.078156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335394304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017697.078159:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017697.078164:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017697.078168:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017697.078170:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017697.078173:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1961 00000100:00080000:2.0:1734017697.078177:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017697.078180:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078182:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86dd880 x1818247335393664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017697.078184:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017697.078193:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335394432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017697.078211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335393664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078222:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078231:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335393664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017697.078239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017697.078257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86df100 x1818247335393792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017697.078274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335393792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078285:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017697.078294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017697.078308:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86dc700 x1818247335393920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017697.078324:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335393920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078331:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078334:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078337:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335393920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017697.078363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017697.078376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86dca80 x1818247335394048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017697.078393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335394048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017697.078412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335394176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017697.078424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86dc380 x1818247335394176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017697.078439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335394176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335394176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017697.078458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335394304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017697.078474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078481:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86ddf80 x1818247335394304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017697.078491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335394304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335394304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017697.078509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335394432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017697.078521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017697.078526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017697/real 1734017697] req@ffff8800a86dfb80 x1818247335394432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017697.078536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335394432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017697.078544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017697.078546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017697.078549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335394432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017697.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017697.367985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017698.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017698.369884:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017699.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017699.372010:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017700.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017700.373943:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017701.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017701.375979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017702.085826:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017702.085836:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017702.085841:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.085851:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.085862:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017702.085870:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017702.085907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335394560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017702.085941:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017702.085945:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017702.085947:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.085952:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.085956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017702.085960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017702.085970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017702.085981:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335394688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017702.085991:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017702.085994:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017702.085996:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.086000:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.086002:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017702.086006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017702.086010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017702.086018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335394816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017702.086026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017702.086029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017702.086030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.086034:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.086036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017702.086040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017702.086044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017702.086053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335394944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017702.086056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335394560:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017702.086068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017702.086071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017702.086072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.086076:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.086078:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017702.086082:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017702.086086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017702.086093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335395072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017702.086101:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017702.086103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017702.086105:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.086108:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.086119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734017702.086124:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017702.086124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017702.086128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017702.086135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335395200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017702.086143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017702.086145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86df480 x1818247335394560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017702.086146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017702.086148:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017702.086152:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1966 00000100:00080000:2.0:1734017702.086155:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017702.086159:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017702.086163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017702.086170:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335395328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017702.086173:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335394560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086183:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086186:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086191:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335394560:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017702.086198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017702.086215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017702.086221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86dd500 x1818247335394688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017702.086232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335394688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017702.086251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017702.086264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017702.086270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86de680 x1818247335394816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017702.086280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335394816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017702.086322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017702.086334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017702.086340:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86df800 x1818247335394944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017702.086350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335394944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335394944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017702.086368:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017702.086379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017702.086385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86ddc00 x1818247335395072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017702.086395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335395072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017702.086413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017702.086427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017702.086434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86dc000 x1818247335395200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017702.086444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335395200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017702.086461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017702.086472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017702.086478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017702/real 1734017702] req@ffff8800a86dd180 x1818247335395328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017702.086488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335395328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017702.086495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017702.086497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017702.086500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017702.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017702.377945:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017703.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017703.379951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017704.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017704.381951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017705.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017705.383986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017706.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017706.385987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017707.093829:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.093837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017707.093842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.093852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00080000:2.0:1734017707.093861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.093868:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017707.093903:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335395456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017707.093936:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.093941:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017707.093943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.093947:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00080000:2.0:1734017707.093950:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.093954:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017707.093963:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017707.093975:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335395584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017707.093984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.093988:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017707.093990:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.093993:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00080000:2.0:1734017707.093996:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.093999:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017707.094004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017707.094011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335395712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017707.094020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.094023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017707.094024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.094027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00080000:2.0:1734017707.094030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.094033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017707.094037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017707.094046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335395456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017707.094046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335395840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017707.094055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.094058:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017707.094060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.094064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00080000:2.0:1734017707.094067:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.094070:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017707.094074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017707.094081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335395968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017707.094089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.094092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017707.094094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.094097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00080000:2.0:1734017707.094101:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.094105:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017707.094108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017707.094112:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017707.094116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335396096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017707.094124:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017707.094127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017707.094128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017707.094131:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1971 00000100:00100000:1.0:1734017707.094134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86de300 x1818247335395456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017707.094134:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017707.094137:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017707.094142:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017707.094149:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335396224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017707.094161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335395456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094171:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094175:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335395456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017707.094186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335395584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017707.094202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017707.094209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86dce00 x1818247335395584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017707.094220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335395584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335395584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017707.094238:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335395712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017707.094251:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017707.094257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86dea00 x1818247335395712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017707.094267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335395712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335395712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017707.094306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335395840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017707.094318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017707.094325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86dd180 x1818247335395840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017707.094335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335395840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094344:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335395840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017707.094352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017707.094364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017707.094370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86dc000 x1818247335395968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017707.094380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335395968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335395968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017707.094397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017707.094412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017707.094418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86ddc00 x1818247335396096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017707.094429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335396096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094438:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017707.094494:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017707.094510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017707.094518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017707/real 1734017707] req@ffff8800a86df800 x1818247335396224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017707.094528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335396224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017707.094535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017707.094537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017707.094540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017707.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017707.387950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017708.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017708.389952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017709.213770:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017709.391989:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017710.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017710.393955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017711.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017711.395963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017712.101830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.101843:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017712.101849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.101863:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.101875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.101885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017712.101932:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335396352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017712.101972:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.101976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017712.101978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.101983:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.101987:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.101991:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017712.102002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017712.102014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335396480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017712.102024:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.102027:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017712.102030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.102034:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.102037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.102041:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017712.102045:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017712.102055:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335396608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017712.102063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.102066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017712.102068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.102071:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.102074:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.102077:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017712.102081:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017712.102093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335396736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017712.102098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335396352:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017712.102102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.102106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017712.102107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.102111:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.102114:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.102117:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017712.102122:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017712.102130:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335396864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017712.102140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.102143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017712.102145:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.102148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.102151:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.102155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017712.102159:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017712.102167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335396992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017712.102171:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017712.102175:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017712.102178:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017712.102179:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017712.102183:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1976 00000100:00080000:2.0:1734017712.102188:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017712.102192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102194:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86de680 x1818247335396352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017712.102196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017712.102204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335397120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017712.102223:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335396352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335396352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017712.102250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335396480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017712.102268:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86dd500 x1818247335396480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017712.102286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335396480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335396480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017712.102307:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335396608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017712.102320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86df480 x1818247335396608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017712.102337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335396608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102347:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335396608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017712.102408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335396736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017712.102425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86dfb80 x1818247335396736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017712.102443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335396736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335396736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017712.102463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017712.102476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86ddf80 x1818247335396864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017712.102492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335396864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017712.102511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017712.102526:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86dc380 x1818247335396992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017712.102541:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335396992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335396992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017712.102560:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335397120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017712.102572:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017712.102578:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017712/real 1734017712] req@ffff8800a86dca80 x1818247335397120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017712.102588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335397120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017712.102596:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017712.102598:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017712.102601:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335397120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017712.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017712.397980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017713.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017713.399968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017714.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017714.401991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017715.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017715.403986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017716.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017716.405976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017717.109837:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.109849:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017717.109854:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.109864:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00080000:2.0:1734017717.109873:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.109881:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017717.109915:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335397248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017717.109949:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.109953:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017717.109955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.109959:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00080000:2.0:1734017717.109962:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.109966:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017717.109975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017717.109986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335397376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017717.109995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.109998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017717.110000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.110003:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00080000:2.0:1734017717.110006:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.110009:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017717.110013:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017717.110021:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335397504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017717.110030:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.110033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017717.110035:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.110038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00080000:2.0:1734017717.110041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.110044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017717.110048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017717.110057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335397248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017717.110057:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335397632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017717.110066:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.110069:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017717.110071:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.110074:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00080000:2.0:1734017717.110076:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.110079:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017717.110083:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017717.110091:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335397760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017717.110099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.110102:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017717.110104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.110107:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00080000:2.0:1734017717.110112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.110116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017717.110120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017717.110123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017717.110127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335397888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017717.110135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017717.110138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017717.110139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017717.110143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1981 00000100:00100000:1.0:1734017717.110145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86dc700 x1818247335397248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017717.110145:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017717.110149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017717.110153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017717.110160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335398016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017717.110172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335397248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110182:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110185:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110190:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335397248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017717.110197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335397376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017717.110213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017717.110220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86df100 x1818247335397376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017717.110231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335397376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110240:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110244:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335397376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017717.110250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335397504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017717.110263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017717.110269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86dd880 x1818247335397504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017717.110332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335397504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335397504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017717.110372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335397632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017717.110388:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017717.110395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86dca80 x1818247335397632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017717.110406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335397632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335397632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017717.110425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335397760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017717.110437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017717.110443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86dc380 x1818247335397760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017717.110452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335397760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335397760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017717.110470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335397888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017717.110485:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017717.110491:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86ddf80 x1818247335397888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017717.110501:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335397888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110509:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110511:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335397888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017717.110519:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017717.110530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017717.110536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017717/real 1734017717] req@ffff8800a86dfb80 x1818247335398016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017717.110546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335398016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017717.110553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017717.110555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017717.110558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017717.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017717.407950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017718.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017718.409951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017719.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017719.411965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017720.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017720.413953:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017721.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017721.415917:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017722.117831:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.117840:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017722.117845:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.117855:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00080000:2.0:1734017722.117864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.117871:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017722.117908:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335398144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017722.117943:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.117948:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017722.117949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.117954:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00080000:2.0:1734017722.117957:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.117960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017722.117970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017722.117981:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335398272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017722.117990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.117994:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017722.117995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.117999:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00080000:2.0:1734017722.118001:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.118004:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017722.118009:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017722.118017:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335398400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017722.118020:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335398144:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017722.118026:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.118029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017722.118031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.118035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00080000:2.0:1734017722.118037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.118040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017722.118045:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017722.118053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335398528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017722.118061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.118064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017722.118066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.118069:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00080000:2.0:1734017722.118072:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.118075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017722.118079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017722.118086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335398656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017722.118088:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017722.118094:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.118096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017722.118098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.118101:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00080000:2.0:1734017722.118105:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.118109:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118110:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86df480 x1818247335398144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017722.118113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017722.118120:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335398784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017722.118129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017722.118132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017722.118134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017722.118137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1986 00000100:00100000:1.0:1734017722.118138:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335398144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017722.118140:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017722.118143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017722.118147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734017722.118149:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118152:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017722.118154:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335398912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017722.118157:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335398144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017722.118164:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335398272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017722.118180:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118223:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86dd500 x1818247335398272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017722.118237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335398272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017722.118245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017722.118252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335398272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017722.118275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335398400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017722.118293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86de680 x1818247335398400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017722.118310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335398400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017722.118318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017722.118324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335398400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017722.118330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335398528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017722.118342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86df800 x1818247335398528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017722.118358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335398528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017722.118365:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017722.118371:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335398528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017722.118376:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017722.118388:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118394:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86ddc00 x1818247335398656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017722.118403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335398656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017722.118411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017722.118416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017722.118422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017722.118437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86dc000 x1818247335398784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017722.118453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335398784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017722.118461:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017722.118465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017722.118471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017722.118482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017722.118488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017722/real 1734017722] req@ffff8800a86dd180 x1818247335398912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017722.118498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335398912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017722.118505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017722.118507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017722.118510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335398912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017722.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017722.417928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017723.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017723.420011:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017724.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017724.421977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017725.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017725.423925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017726.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017726.425942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017727.125836:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.125845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017727.125849:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017727.125859:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.125867:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.125875:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017727.125912:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335399040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017727.125947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.125951:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017727.125954:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017727.125958:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.125961:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.125965:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017727.125975:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017727.125985:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335399168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017727.125995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.125998:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017727.126008:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017727.126012:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.126015:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.126018:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017727.126023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017727.126031:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335399296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017727.126039:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.126042:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017727.126044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017727.126047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.126050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.126053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017727.126057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017727.126066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335399424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017727.126073:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335399040:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017727.126076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.126079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017727.126080:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017727.126084:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.126086:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.126090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017727.126094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017727.126101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335399552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017727.126141:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.126144:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017727.126146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017727.126149:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.126152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.126155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017727.126160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017727.126168:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335399680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017727.126176:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017727.126179:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017727.126181:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017727.126183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017727.126184:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1991 00000100:00080000:2.0:1734017727.126188:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017727.126191:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017727.126195:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017727.126202:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335399808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017727.126205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86dea00 x1818247335399040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335399040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126246:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335399040:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017727.126259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335399168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017727.126277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017727.126284:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86dce00 x1818247335399168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335399168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335399168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017727.126314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335399296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017727.126327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017727.126333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86de300 x1818247335399296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335399296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335399296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017727.126381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335399424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017727.126394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017727.126400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86dd180 x1818247335399424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335399424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335399424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017727.126429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335399552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017727.126440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017727.126446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86dc000 x1818247335399552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335399552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335399552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017727.126474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335399680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017727.126489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017727.126495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86ddc00 x1818247335399680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335399680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335399680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017727.126522:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335399808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017727.126534:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017727.126540:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017727/real 1734017727] req@ffff8800a86df800 x1818247335399808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017727.126549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335399808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017727.126556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017727.126558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017727.126561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335399808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017727.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017727.427960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017728.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017728.429957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017729.213795:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017729.431894:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017730.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017730.433939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017731.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017731.435962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017732.133839:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.133850:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017732.133855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.133865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.133875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.133883:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017732.133918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335399936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017732.133953:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.133957:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017732.133960:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.133964:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.133967:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.133971:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017732.133981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017732.133997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335400064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017732.134007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.134010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017732.134012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.134015:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.134043:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.134047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017732.134052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017732.134061:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335400192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017732.134070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.134073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017732.134075:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.134078:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.134081:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.134084:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017732.134088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017732.134093:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335399936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017732.134097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335400320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017732.134106:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.134110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017732.134111:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.134116:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.134118:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.134121:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017732.134125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017732.134132:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335400448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017732.134140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.134143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017732.134145:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.134148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.134151:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.134154:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017732.134158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017732.134165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335400576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017732.134172:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017732.134175:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017732.134177:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017732.134180:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 1996 00000100:00080000:2.0:1734017732.134183:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017732.134186:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017732.134188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400064:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734017732.134190:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017732.134197:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335400704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017732.134210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017732.134234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017732.134250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335400448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017732.134266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335400576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017732.134280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335400704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017732.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017732.437974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017733.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017733.439896:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017734.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017734.441969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1734017735.140070:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017735.140133:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017735.140157:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017735.140209:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017735.140223:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017735.140234:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017735.140246:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017735.140257:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017735.140268:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017735.140337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86dfb80 x1818247335400320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140376:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335400320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017735.140421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86ddf80 x1818247335400448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335400448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335400448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017735.140487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86dc380 x1818247335400576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335400576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335400576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017735.140537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86dca80 x1818247335400704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335400704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140567:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335400704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017735.140586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86de680 x1818247335399936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140605:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335399936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140621:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140626:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335399936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017735.140636:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86dd500 x1818247335400064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140655:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335400064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140678:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140682:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140687:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017735.140903:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017732/real 1734017735] req@ffff8800a86df480 x1818247335400192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017735.140918:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335400192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017735.140926:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017735.140929:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017735.140932:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017735.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017735.443942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017736.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017736.445921:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017737.141807:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017737.141817:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017737.141822:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017737.141833:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2001 00000100:00080000:2.0:1734017737.141842:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017737.141849:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017737.141884:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335400832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017737.141919:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017737.141923:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017737.141925:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017737.141930:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2001 00000100:00080000:2.0:1734017737.141977:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017737.141982:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017737.141993:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017737.142005:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335400960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017737.142016:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017737.142019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1734017737.142038:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335400832:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017737.142060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017737.142065:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2001 00000100:00080000:2.0:1734017737.142068:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017737.142071:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017737.142077:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017737.142086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335401088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017737.142096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017737.142100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017737.142101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017737.142105:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2001 00000800:00000100:1.0:1734017737.142107:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017737.142108:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017737.142111:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017737.142115:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017737.142125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335401216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017737.142128:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86dd880 x1818247335400832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017737.142134:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017737.142137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017737.142138:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017737.142142:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2001 00000100:00080000:2.0:1734017737.142145:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017737.142147:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335400832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017737.142149:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017737.142153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1734017737.142158:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1734017737.142160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335401344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017737.142161:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017737.142166:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335400832:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017737.142170:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017737.142173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400960:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734017737.142173:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017737.142174:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017737.142178:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2001 00000100:00080000:2.0:1734017737.142180:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017737.142183:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017737.142187:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017737.142189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017737.142195:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335401472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017737.142197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86df100 x1818247335400960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017737.142203:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017737.142205:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017737.142207:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017737.142208:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335400960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017737.142211:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2001 00000100:00080000:2.0:1734017737.142213:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734017737.142216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734017737.142216:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:1.0:1734017737.142218:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734017737.142220:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017737.142222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335400960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017737.142227:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335401600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017737.142245:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017737.142259:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017737.142266:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86dc700 x1818247335401088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017737.142276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335401088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017737.142283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017737.142285:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017737.142289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017737.142295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017737.142306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017737.142312:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86df480 x1818247335401216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017737.142321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335401216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017737.142336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017737.142339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017737.142342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017737.142348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335401344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017737.142360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017737.142365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86dd500 x1818247335401344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017737.142375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335401344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017737.142382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017737.142385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017737.142389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335401344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017737.142394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335401472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017737.142406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017737.142412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86de680 x1818247335401472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017737.142421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335401472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017737.142429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017737.142431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017737.142434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335401472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017737.142441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335401600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017737.142453:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017737.142458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017737/real 1734017737] req@ffff8800a86dca80 x1818247335401600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017737.142468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335401600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017737.142475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017737.142477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017737.142480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335401600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017737.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017737.448016:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017738.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017738.449990:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017739.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017739.451985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017740.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017740.453959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017741.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017741.456007:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017742.149905:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.149918:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017742.149924:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.149938:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.149949:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.149959:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017742.150007:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335401728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017742.150045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.150050:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017742.150052:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.150057:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.150061:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.150064:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017742.150076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017742.150087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335401856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017742.150097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.150100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017742.150102:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.150106:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.150109:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.150113:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017742.150117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017742.150126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335401984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017742.150136:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.150139:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017742.150141:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.150145:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.150147:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.150151:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017742.150155:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017742.150167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335402112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017742.150174:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335401728:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017742.150176:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.150180:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017742.150182:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.150185:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.150188:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.150192:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017742.150196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017742.150205:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335402240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017742.150213:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.150216:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017742.150218:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.150221:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.150225:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.150229:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017742.150234:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017742.150241:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335402368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017742.150249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017742.150249:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017742.150252:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017742.150253:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017742.150257:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2006 00000100:00080000:2.0:1734017742.150261:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017742.150264:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017742.150268:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017742.150273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86dc380 x1818247335401728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017742.150276:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335402496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017742.150295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335401728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335401728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017742.150324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017742.150342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017742.150349:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86ddf80 x1818247335401856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017742.150368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335401856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017742.150389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017742.150403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017742.150409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86dfb80 x1818247335401984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017742.150419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335401984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335401984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017742.150459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335402112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017742.150472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017742.150478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86df800 x1818247335402112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017742.150488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335402112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335402112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017742.150509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335402240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017742.150521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017742.150527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86ddc00 x1818247335402240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017742.150537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335402240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335402240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017742.150556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335402368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017742.150568:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017742.150574:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86dc000 x1818247335402368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017742.150584:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335402368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150592:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150594:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335402368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017742.150602:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335402496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017742.150618:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017742.150624:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017742/real 1734017742] req@ffff8800a86dd180 x1818247335402496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017742.150634:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335402496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017742.150641:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017742.150643:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017742.150646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335402496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017742.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017742.457983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017743.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017743.459975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017744.213908:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017744.461977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017745.213887:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017745.463940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017746.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017746.466004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017747.157849:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.157858:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017747.157863:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017747.157873:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.157882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.157890:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017747.157927:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335402624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017747.157959:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.157964:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017747.157965:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017747.157970:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.157973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.157976:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017747.157986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017747.157996:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335402752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017747.158006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.158009:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017747.158011:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017747.158014:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.158017:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.158020:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017747.158025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017747.158033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335402880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017747.158041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.158044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017747.158045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017747.158050:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.158052:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.158055:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017747.158060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017747.158067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335402624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017747.158069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335403008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017747.158078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.158081:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017747.158083:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017747.158086:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.158089:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.158092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017747.158096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017747.158104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335403136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017747.158112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.158115:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017747.158116:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017747.158120:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.158122:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.158125:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017747.158129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017747.158132:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017747.158136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335403264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017747.158145:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017747.158149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017747.158150:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017747.158153:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86de300 x1818247335402624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017747.158154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2011 00000100:00080000:2.0:1734017747.158156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017747.158159:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017747.158163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017747.158171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335403392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017747.158172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335402624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158183:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158186:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158191:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335402624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017747.158198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335402752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017747.158215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017747.158221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86dce00 x1818247335402752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017747.158232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335402752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335402752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017747.158252:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335402880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017747.158266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017747.158272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86dea00 x1818247335402880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017747.158282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335402880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158291:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335402880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017747.158318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017747.158331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017747.158337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86dd180 x1818247335403008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017747.158347:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335403008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158355:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017747.158366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335403136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017747.158378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017747.158384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86dc000 x1818247335403136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017747.158394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335403136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335403136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017747.158412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335403264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017747.158423:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017747.158429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86ddc00 x1818247335403264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017747.158439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335403264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335403264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017747.158457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335403392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017747.158469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017747.158474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017747/real 1734017747] req@ffff8800a86df800 x1818247335403392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017747.158485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335403392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017747.158492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017747.158494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017747.158497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335403392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017747.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017747.467997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017748.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017748.469966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017749.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017749.471947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017750.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017750.473912:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017751.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017751.475962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017752.165847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.165856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017752.165860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.165871:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00080000:2.0:1734017752.165880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.165887:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017752.165922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335403520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017752.165955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.165960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017752.165963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.165967:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00080000:2.0:1734017752.165970:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.165974:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017752.165984:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017752.165994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335403648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017752.166003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.166007:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017752.166008:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.166012:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00080000:2.0:1734017752.166015:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.166018:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017752.166022:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017752.166030:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335403776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017752.166038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.166041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017752.166043:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.166046:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00080000:2.0:1734017752.166048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.166052:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017752.166055:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017752.166060:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335403520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017752.166065:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335403904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017752.166074:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.166078:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017752.166079:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.166083:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00080000:2.0:1734017752.166085:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.166088:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017752.166092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017752.166100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335404032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017752.166107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.166110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017752.166112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.166115:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00080000:2.0:1734017752.166118:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.166121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017752.166125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017752.166127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017752.166132:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335404160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017752.166139:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017752.166142:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017752.166144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017752.166147:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2016 00000100:00100000:1.0:1734017752.166148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86dfb80 x1818247335403520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017752.166151:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017752.166154:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017752.166158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017752.166166:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335404288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017752.166168:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335403520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166178:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166181:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166186:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335403520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017752.166194:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017752.166210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017752.166217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86ddf80 x1818247335403648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017752.166228:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335403648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166236:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166239:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017752.166249:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017752.166263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017752.166269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86dc380 x1818247335403776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017752.166279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335403776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017752.166316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017752.166329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017752.166335:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86dca80 x1818247335403904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017752.166345:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335403904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335403904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017752.166364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017752.166379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017752.166385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86de680 x1818247335404032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017752.166395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335404032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017752.166413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017752.166424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017752.166430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86dd500 x1818247335404160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017752.166440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335404160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017752.166458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017752.166469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017752.166475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017752/real 1734017752] req@ffff8800a86df480 x1818247335404288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017752.166491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335404288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017752.166499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017752.166501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017752.166504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017752.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017752.477949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017753.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017753.479956:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017754.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017754.481940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017755.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017755.483948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017756.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017756.485912:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017757.173845:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.173854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017757.173858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017757.173868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.173881:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.173888:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017757.173923:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335404416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017757.173958:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.173963:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017757.173965:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017757.173969:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.173973:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.173976:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017757.173986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017757.173997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335404544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017757.174006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.174010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017757.174012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017757.174015:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.174018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.174021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017757.174025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017757.174033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335404672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017757.174041:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.174044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017757.174045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017757.174049:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.174051:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.174054:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017757.174058:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017757.174061:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335404416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017757.174067:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335404800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017757.174076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.174079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017757.174080:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017757.174084:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.174086:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.174090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017757.174094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017757.174101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335404928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017757.174109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.174112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017757.174113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017757.174117:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.174119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.174123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017757.174126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017757.174127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017757.174134:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335405056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017757.174142:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017757.174144:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017757.174146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017757.174147:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86dc700 x1818247335404416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017757.174150:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2021 00000100:00080000:2.0:1734017757.174152:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017757.174155:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017757.174159:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017757.174166:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335405184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017757.174167:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335404416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174178:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174181:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174186:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335404416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017757.174193:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335404544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017757.174209:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017757.174216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86df100 x1818247335404544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017757.174227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335404544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174235:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335404544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017757.174247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335404672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017757.174261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017757.174267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86dd880 x1818247335404672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017757.174277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335404672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335404672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017757.174314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335404800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017757.174326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017757.174333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86df480 x1818247335404800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017757.174343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335404800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335404800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017757.174361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017757.174376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017757.174382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86dd500 x1818247335404928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017757.174392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335404928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335404928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017757.174411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017757.174422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017757.174428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86de680 x1818247335405056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017757.174437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335405056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017757.174455:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017757.174468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017757.174473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017757/real 1734017757] req@ffff8800a86dca80 x1818247335405184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017757.174489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335405184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017757.174496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017757.174498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017757.174501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017757.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017757.487958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017758.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017758.489968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017759.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017759.491980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017760.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017760.493949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017761.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017761.495977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017762.181844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017762.181855:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017762.181861:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017762.181873:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.181884:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.181894:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017762.181942:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335405312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017762.181983:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017762.181988:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017762.181990:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017762.181995:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.181999:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.182002:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017762.182014:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017762.182025:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335405440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017762.182035:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017762.182038:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017762.182041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017762.182046:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.182048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.182052:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017762.182057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017762.182066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335405568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017762.182075:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017762.182078:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017762.182079:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017762.182084:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.182086:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.182089:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017762.182093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017762.182104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335405696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017762.182114:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017762.182118:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335405312:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017762.182118:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017762.182119:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017762.182123:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.182126:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.182129:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017762.182134:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017762.182142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335405824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017762.182152:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017762.182155:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017762.182157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017762.182161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.182163:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.182166:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017762.182171:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017762.182178:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335405952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017762.182186:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017762.182189:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017762.182190:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017762.182193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017762.182194:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2026 00000100:00080000:2.0:1734017762.182198:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017762.182201:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017762.182205:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017762.182213:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335406080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017762.182217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86dc380 x1818247335405312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335405312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335405312:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017762.182267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335405440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017762.182284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017762.182291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86ddf80 x1818247335405440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335405440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335405440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017762.182324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335405568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017762.182339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017762.182345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86dfb80 x1818247335405568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335405568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335405568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017762.182396:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335405696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017762.182408:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017762.182414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86df800 x1818247335405696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335405696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182433:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182435:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182438:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335405696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017762.182444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017762.182460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017762.182466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86ddc00 x1818247335405824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335405824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017762.182495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017762.182555:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017762.182564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86dc000 x1818247335405952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335405952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335405952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017762.182594:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017762.182610:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017762.182616:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017762/real 1734017762] req@ffff8800a86dd180 x1818247335406080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017762.182633:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335406080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017762.182641:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017762.182643:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017762.182646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017762.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017762.497997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017763.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017763.499981:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017764.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017764.501992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017765.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017765.503996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017766.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017766.505976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017767.189790:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017767.189800:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017767.189804:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.189815:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.189825:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.189833:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017767.189867:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335406208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017767.189902:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017767.189906:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017767.189908:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.189912:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.189916:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.189926:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017767.189935:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017767.189946:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335406336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017767.189956:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017767.189959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017767.189961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.189964:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.189967:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.189970:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017767.189974:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017767.189982:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335406464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017767.189990:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335406208:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017767.189992:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017767.189996:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017767.189997:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.190001:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.190003:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.190006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017767.190010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017767.190020:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335406592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017767.190028:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017767.190031:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017767.190032:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.190036:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.190039:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.190042:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017767.190046:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017767.190053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335406720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017767.190059:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017767.190061:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017767.190064:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017767.190066:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.190069:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.190075:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.190078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190080:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86dea00 x1818247335406208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017767.190082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017767.190089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335406848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017767.190097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017767.190100:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335406208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017767.190100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017767.190101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017767.190105:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2031 00000100:00080000:2.0:1734017767.190107:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017767.190110:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:1.0:1734017767.190111:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1734017767.190114:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734017767.190115:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190120:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335406208:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017767.190121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335406976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017767.190127:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335406336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017767.190145:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190152:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86dce00 x1818247335406336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017767.190163:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335406336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017767.190171:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017767.190174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190178:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335406336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017767.190201:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335406464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017767.190216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86de300 x1818247335406464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017767.190233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335406464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017767.190240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017767.190242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335406464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017767.190251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335406592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017767.190262:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86dd180 x1818247335406592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017767.190278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335406592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017767.190285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017767.190287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335406592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017767.190296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017767.190312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86dc000 x1818247335406720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017767.190328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335406720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017767.190335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017767.190338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017767.190346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017767.190358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86ddc00 x1818247335406848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017767.190374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335406848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017767.190381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017767.190383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017767.190392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017767.190404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017767.190409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017767/real 1734017767] req@ffff8800a86df800 x1818247335406976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017767.190462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335406976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017767.190470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017767.190472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017767.190476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335406976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017767.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017767.507979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017768.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017768.509966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017769.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017769.511859:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017770.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017770.513927:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017771.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017771.515979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017772.197838:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.197847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017772.197851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.197862:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.197871:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.197880:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017772.197916:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335407104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017772.197950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.197955:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017772.197956:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.197961:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.197964:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.197968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017772.197977:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017772.197988:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335407232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017772.197998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.198001:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017772.198003:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.198006:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.198009:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.198012:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017772.198017:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017772.198024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335407360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017772.198033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.198036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017772.198037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.198041:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.198044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.198047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017772.198051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017772.198060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335407488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017772.198064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335407104:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017772.198069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.198072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017772.198074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.198077:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.198080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.198083:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017772.198088:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017772.198095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335407616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017772.198104:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.198107:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017772.198108:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.198111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.198114:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.198117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017772.198121:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017772.198128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335407744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017772.198132:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017772.198137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017772.198140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017772.198141:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017772.198145:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2036 00000100:00080000:2.0:1734017772.198149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017772.198153:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198154:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86dfb80 x1818247335407104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017772.198156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017772.198164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335407872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017772.198173:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335407104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198185:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198188:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198193:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335407104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017772.198200:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335407232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017772.198217:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86ddf80 x1818247335407232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017772.198235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335407232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198242:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198246:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335407232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017772.198255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335407360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017772.198269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86dc380 x1818247335407360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017772.198285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335407360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335407360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017772.198323:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335407488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017772.198380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198390:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86dca80 x1818247335407488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017772.198401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335407488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198411:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198414:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335407488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017772.198420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335407616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017772.198440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86de680 x1818247335407616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017772.198457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335407616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335407616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017772.198475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335407744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017772.198487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86dd500 x1818247335407744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017772.198502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335407744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335407744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017772.198520:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335407872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017772.198532:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017772.198538:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017772/real 1734017772] req@ffff8800a86df480 x1818247335407872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017772.198555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335407872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017772.198562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017772.198564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017772.198567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335407872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017772.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017772.517959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017773.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017773.519977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017774.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017774.521955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017775.213795:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017775.523946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017776.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017776.525929:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017777.205833:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017777.205841:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017777.205846:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.205856:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.205865:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.205872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017777.205907:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335408000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017777.205940:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017777.205944:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017777.205947:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.205952:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.205955:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.205958:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017777.205968:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017777.205978:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335408128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017777.205988:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017777.205991:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017777.205993:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.205997:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.206000:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.206003:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017777.206007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017777.206015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335408256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017777.206023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017777.206026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017777.206027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.206030:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.206033:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.206037:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017777.206040:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017777.206050:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335408000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017777.206051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335408384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017777.206060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017777.206063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017777.206065:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.206068:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.206070:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.206073:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017777.206077:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017777.206085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335408512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017777.206093:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017777.206096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017777.206097:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.206100:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.206112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.206116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017777.206119:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017777.206120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017777.206128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335408640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017777.206136:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017777.206139:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86dd880 x1818247335408000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017777.206139:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017777.206140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017777.206144:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2041 00000100:00080000:2.0:1734017777.206148:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017777.206151:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017777.206155:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017777.206160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335408000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017777.206162:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335408768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017777.206170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335408000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017777.206186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335408128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017777.206202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017777.206209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86df100 x1818247335408128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017777.206220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335408128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017777.206228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335408128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017777.206241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335408256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017777.206292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017777.206300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86dc700 x1818247335408256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017777.206311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335408256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017777.206318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335408256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017777.206353:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335408384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017777.206367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017777.206373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86df480 x1818247335408384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017777.206383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335408384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017777.206391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335408384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017777.206401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335408512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017777.206417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017777.206423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86dd500 x1818247335408512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017777.206433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335408512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017777.206441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206446:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335408512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017777.206452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335408640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017777.206463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017777.206468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86de680 x1818247335408640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017777.206478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335408640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017777.206485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335408640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017777.206496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335408768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017777.206507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017777.206513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017777/real 1734017777] req@ffff8800a86dca80 x1818247335408768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017777.206530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335408768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017777.206537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017777.206539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017777.206542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335408768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017777.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017777.527987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017778.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017778.529972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017779.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017779.531981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017780.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017780.533960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017781.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017781.535967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734017782.213824:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.213833:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017782.213838:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.213848:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.213857:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.213864:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017782.213899:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335408896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734017782.213904:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017782.213928:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.213932:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017782.213934:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.213938:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.213941:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.213945:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017782.213954:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017782.213965:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335409024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017782.213975:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.213978:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017782.213980:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.213983:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.213985:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.213988:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017782.213993:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017782.214001:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335409152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017782.214009:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.214012:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017782.214013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.214017:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.214019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.214023:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017782.214027:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017782.214036:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335409280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017782.214044:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335408896:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017782.214045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.214048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017782.214050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.214055:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.214057:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.214060:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017782.214064:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017782.214072:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335409408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017782.214079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.214082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017782.214084:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.214087:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.214089:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.214093:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017782.214096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017782.214103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335409536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017782.214111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017782.214114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1734017782.214115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017782.214115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017782.214119:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2046 00000100:00080000:2.0:1734017782.214122:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017782.214125:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017782.214129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017782.214135:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86dc380 x1818247335408896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017782.214136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335409664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017782.214155:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335408896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214212:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335408896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017782.214229:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017782.214250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017782.214258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86ddf80 x1818247335409024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017782.214269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335409024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017782.214291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017782.214305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017782.214311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86dfb80 x1818247335409152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017782.214321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335409152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017782.214358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017782.214370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017782.214376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86df800 x1818247335409280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017782.214387:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335409280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017782.214404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335409408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017782.214419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017782.214426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86ddc00 x1818247335409408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017782.214436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335409408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214449:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335409408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017782.214454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335409536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017782.214465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017782.214471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86dc000 x1818247335409536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017782.214481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335409536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335409536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017782.214498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335409664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017782.214510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017782.214516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017782/real 1734017782] req@ffff8800a86dd180 x1818247335409664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017782.214532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335409664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017782.214540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017782.214542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017782.214545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335409664:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017782.537954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017783.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017783.539944:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017784.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017784.541978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017785.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017785.543928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017786.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017786.545970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017787.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017787.221838:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017787.221847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017787.221853:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.221865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.221876:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.221885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017787.221934:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335409792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017787.221972:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017787.221976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017787.221978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.221982:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.221986:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.221990:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017787.222001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017787.222012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335409920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017787.222021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017787.222025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017787.222026:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.222031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.222034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.222037:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017787.222044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017787.222053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335410048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017787.222062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017787.222066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017787.222067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.222071:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.222073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.222077:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017787.222113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017787.222127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335410176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017787.222137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017787.222139:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335409792:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017787.222140:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017787.222142:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.222146:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.222149:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.222153:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017787.222158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017787.222167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335410304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017787.222176:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017787.222179:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017787.222182:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.222186:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.222188:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.222192:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017787.222196:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017787.222204:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335410432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017787.222212:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017787.222215:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1734017787.222216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017787.222216:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017787.222220:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2051 00000100:00080000:2.0:1734017787.222223:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017787.222227:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017787.222230:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017787.222238:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335410560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017787.222241:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86de300 x1818247335409792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335409792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335409792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017787.222289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017787.222307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017787.222314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86dce00 x1818247335409920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335409920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222333:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335409920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017787.222347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017787.222361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017787.222367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86dea00 x1818247335410048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335410048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017787.222418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017787.222431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017787.222437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86dd180 x1818247335410176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335410176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222461:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017787.222467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335410304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017787.222484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017787.222490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86dc000 x1818247335410304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335410304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335410304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017787.222518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335410432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017787.222530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017787.222536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86ddc00 x1818247335410432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335410432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222554:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222557:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222560:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335410432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017787.222566:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335410560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017787.222578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017787.222584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017787/real 1734017787] req@ffff8800a86df800 x1818247335410560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017787.222600:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335410560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017787.222608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017787.222610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017787.222613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335410560:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017787.547977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017788.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017788.549974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017789.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017789.551954:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017790.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017790.553949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017791.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017791.555981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017792.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017792.229838:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.229847:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017792.229852:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017792.229862:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00080000:2.0:1734017792.229872:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017792.229880:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017792.229914:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335410688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017792.229955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.229960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017792.229961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017792.229966:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00080000:2.0:1734017792.229969:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017792.229973:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017792.229982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017792.229992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335410816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017792.230045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.230048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017792.230050:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017792.230054:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00080000:2.0:1734017792.230057:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017792.230060:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017792.230065:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017792.230074:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335410944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017792.230084:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.230087:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017792.230089:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017792.230092:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00100000:1.0:1734017792.230093:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335410688:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017792.230095:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017792.230099:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017792.230103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017792.230113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335411072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017792.230121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.230124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017792.230126:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017792.230129:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00080000:2.0:1734017792.230131:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017792.230135:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017792.230139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017792.230147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335411200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017792.230155:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.230158:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017792.230159:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017792.230162:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017792.230163:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00080000:2.0:1734017792.230165:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017792.230169:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017792.230172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017792.230179:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335411328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017792.230183:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86dfb80 x1818247335410688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017792.230187:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017792.230191:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017792.230193:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017792.230196:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2056 00000100:00080000:2.0:1734017792.230200:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017792.230201:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335410688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017792.230203:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017792.230207:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734017792.230212:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1734017792.230215:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335411456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017792.230216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230220:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335410688:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017792.230228:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017792.230245:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017792.230251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86ddf80 x1818247335410816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017792.230263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335410816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017792.230270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017792.230273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017792.230300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017792.230314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017792.230321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86dc380 x1818247335410944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017792.230331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335410944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017792.230339:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017792.230341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335410944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017792.230350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017792.230361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017792.230367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86dca80 x1818247335411072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017792.230377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335411072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017792.230384:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017792.230386:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017792.230395:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335411200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017792.230411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017792.230417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86de680 x1818247335411200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017792.230427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335411200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017792.230434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017792.230436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335411200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017792.230445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335411328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017792.230456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017792.230462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86dd500 x1818247335411328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017792.230472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335411328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017792.230480:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017792.230482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335411328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017792.230491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335411456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017792.230503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017792.230509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017792/real 1734017792] req@ffff8800a86df480 x1818247335411456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017792.230524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335411456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017792.230532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017792.230534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017792.230537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335411456:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017792.557967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017793.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017793.559960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017794.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017794.561990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017795.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017795.563977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017796.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017796.565983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017797.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017797.237794:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017797.237802:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017797.237807:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.237817:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.237826:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.237834:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017797.237871:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335411584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017797.237903:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017797.237934:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017797.237937:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.237942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.237945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.237949:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017797.237958:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017797.237971:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335411712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017797.237982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017797.237986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017797.237987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.237991:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.237993:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.237996:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017797.238001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017797.238008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335411840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017797.238018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017797.238020:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335411584:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017797.238021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017797.238022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.238026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.238028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.238032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017797.238036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017797.238046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335411968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017797.238054:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017797.238057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017797.238059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.238062:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.238066:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.238069:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017797.238073:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017797.238081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335412096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017797.238089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017797.238091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017797.238093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.238096:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.238099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.238102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017797.238106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017797.238113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734017797.238113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335412224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017797.238121:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017797.238124:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017797.238125:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017797.238128:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2061 00000100:00080000:2.0:1734017797.238131:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017797.238134:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017797.238138:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017797.238142:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734017797.238145:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335412352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017797.238159:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017797.238173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017797.238187:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017797.238203:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412352:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017797.567985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017798.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017798.569975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017799.213971:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017799.571969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017800.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:0.0:1734017800.244171:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017800.244236:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017800.244262:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017800.244319:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017800.244329:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017800.244338:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017800.244345:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017800.244352:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017800.244359:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017800.244542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86dca80 x1818247335412352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.244594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335412352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.244613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.244618:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.244627:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017800.244640:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86dd880 x1818247335411840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.244660:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335411840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.244674:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.244678:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.244684:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017800.245439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86df480 x1818247335411968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.245455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335411968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.245464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.245467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.245470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017800.245477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86dd500 x1818247335412096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.245489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335412096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.245497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.245500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.245503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017800.245513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86de680 x1818247335412224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.245524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335412224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.245532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.245535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.245538:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017800.245544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86dc700 x1818247335411584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.245556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335411584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.245563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.245565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.245569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335411584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017800.245575:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017797/real 1734017800] req@ffff8800a86df100 x1818247335411712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017800.245586:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335411712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017800.245593:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017800.245596:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017800.245599:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335411712:192.168.201.104@tcp:38:kworker.0 00010000:00010000:3.0:1734017800.574017:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017801.213921:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017801.576014:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017802.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017802.245821:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017802.245858:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017802.245864:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.245876:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.245887:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.245896:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017802.245943:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335412480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017802.245981:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017802.245986:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017802.245987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.245992:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.245996:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.246000:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017802.246012:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017802.246023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335412608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017802.246032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017802.246036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017802.246037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.246043:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.246045:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.246049:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017802.246054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017802.246063:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335412736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017802.246072:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017802.246075:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017802.246076:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.246080:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.246083:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.246087:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017802.246091:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017802.246102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335412864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017802.246110:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335412480:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017802.246112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017802.246115:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017802.246117:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.246121:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.246123:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.246127:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017802.246131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017802.246140:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335412992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017802.246149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017802.246152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017802.246154:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.246158:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.246160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.246164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017802.246168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017802.246176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335413120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017802.246183:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734017802.246187:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017802.246187:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017802.246188:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017802.246192:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2066 00000100:00080000:2.0:1734017802.246195:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017802.246199:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017802.246203:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017802.246210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86dc380 x1818247335412480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017802.246211:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335413248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017802.246230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335412480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246242:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335412480:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017802.246258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335412608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017802.246276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017802.246282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86ddf80 x1818247335412608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017802.246294:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335412608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335412608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017802.246314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335412736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017802.246327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017802.246334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86dfb80 x1818247335412736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017802.246344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335412736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335412736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017802.246385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335412864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017802.246398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017802.246404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86df800 x1818247335412864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017802.246414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335412864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335412864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017802.246433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017802.246446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017802.246451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86ddc00 x1818247335412992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017802.246461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335412992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335412992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017802.246481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335413120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017802.246493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017802.246499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86dc000 x1818247335413120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017802.246509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335413120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246516:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335413120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017802.246527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335413248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017802.246539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017802.246545:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017802/real 1734017802] req@ffff8800a86dd180 x1818247335413248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017802.246555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335413248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017802.246562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017802.246564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017802.246567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335413248:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017802.577986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017803.213875:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017803.579916:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017804.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017804.581938:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017805.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017805.583965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017806.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017806.585971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017807.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017807.253825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.253834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017807.253839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.253848:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.253857:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.253864:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017807.253899:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335413376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017807.253932:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.253936:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017807.253938:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.253942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.253945:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.253949:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017807.253958:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017807.253968:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335413504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017807.253978:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.253981:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017807.253983:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.253987:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.253989:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.253992:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017807.253997:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017807.254005:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335413632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017807.254015:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.254018:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017807.254020:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.254023:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.254025:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.254028:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017807.254032:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017807.254041:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335413760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017807.254047:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335413376:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017807.254050:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.254053:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017807.254055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.254058:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.254061:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.254064:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017807.254068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017807.254075:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335413888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017807.254084:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.254087:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017807.254088:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.254092:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.254095:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.254099:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017807.254103:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017807.254110:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335414016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017807.254117:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017807.254118:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017807.254121:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017807.254122:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017807.254126:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2071 00000100:00080000:2.0:1734017807.254128:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017807.254131:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017807.254135:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017807.254139:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86dea00 x1818247335413376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017807.254142:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335414144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017807.254158:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335413376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254169:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254172:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254177:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335413376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017807.254185:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335413504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017807.254201:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017807.254208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86dce00 x1818247335413504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017807.254220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335413504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335413504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017807.254239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335413632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017807.254252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017807.254258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86de300 x1818247335413632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017807.254268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335413632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335413632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017807.254306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335413760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017807.254319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017807.254325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86dd180 x1818247335413760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017807.254335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335413760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335413760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017807.254354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335413888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017807.254366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017807.254371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86dc000 x1818247335413888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017807.254381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335413888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254389:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254391:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335413888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017807.254400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017807.254415:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017807.254421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86ddc00 x1818247335414016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017807.254431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335414016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017807.254448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017807.254460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017807.254466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017807/real 1734017807] req@ffff8800a86df800 x1818247335414144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017807.254476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335414144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017807.254483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017807.254485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017807.254488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414144:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017807.587980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017808.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017808.589958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017809.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017809.591952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017810.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017810.593940:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017811.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017811.595960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017812.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017812.261854:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017812.261862:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017812.261867:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.261876:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.261885:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.261893:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017812.261930:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335414272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017812.261963:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017812.261967:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017812.261968:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.261973:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.261976:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.261979:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017812.261989:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017812.261999:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335414400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017812.262008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017812.262012:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017812.262013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.262017:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.262019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.262023:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017812.262027:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017812.262035:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335414528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017812.262043:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017812.262046:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017812.262048:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.262052:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.262054:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.262058:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017812.262062:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017812.262070:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335414656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017812.262079:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017812.262080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335414272:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017812.262082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017812.262084:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.262088:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.262090:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.262093:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017812.262097:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017812.262105:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335414784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017812.262113:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017812.262116:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017812.262117:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.262121:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.262123:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.262127:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017812.262131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017812.262137:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335414912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017812.262146:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734017812.262148:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017812.262149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017812.262151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017812.262154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2076 00000100:00080000:2.0:1734017812.262157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017812.262160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017812.262164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017812.262170:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86dfb80 x1818247335414272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017812.262171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335415040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017812.262197:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335414272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262216:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335414272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017812.262223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335414400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017812.262239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017812.262246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86ddf80 x1818247335414400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017812.262257:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335414400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335414400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017812.262276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335414528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017812.262289:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017812.262296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86dc380 x1818247335414528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017812.262305:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335414528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262313:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335414528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017812.262344:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335414656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017812.262357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017812.262363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86df100 x1818247335414656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017812.262374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335414656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335414656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017812.262392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017812.262403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017812.262409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86dc700 x1818247335414784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017812.262419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335414784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017812.262437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017812.262452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017812.262457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86de680 x1818247335414912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017812.262468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335414912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335414912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017812.262485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017812.262496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017812.262502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017812/real 1734017812] req@ffff8800a86dd500 x1818247335415040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017812.262512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335415040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017812.262519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017812.262521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017812.262524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415040:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017812.597962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017813.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017813.599932:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017814.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017814.601948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017815.213750:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017815.603925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017816.213790:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017816.605950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017817.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017817.269833:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.269842:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017817.269845:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017817.269855:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.269864:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.269872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017817.269904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335415168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017817.269937:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.269942:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017817.269944:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017817.269949:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.269952:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.269956:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017817.269965:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017817.269976:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335415296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017817.269986:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.269989:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017817.269991:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017817.269995:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.269997:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.270001:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017817.270005:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017817.270013:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335415424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017817.270021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.270024:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017817.270026:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017817.270029:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.270031:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.270035:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017817.270038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017817.270046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335415168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017817.270048:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335415552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017817.270058:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.270061:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017817.270062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017817.270066:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.270068:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.270071:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017817.270076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017817.270083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335415680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017817.270091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.270094:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017817.270096:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017817.270099:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.270102:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.270106:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017817.270109:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017817.270112:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017817.270117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335415808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017817.270125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017817.270127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017817.270129:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017817.270133:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86df480 x1818247335415168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017817.270133:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2081 00000100:00080000:2.0:1734017817.270137:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017817.270140:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017817.270144:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017817.270151:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335415936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017817.270161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335415168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270171:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335415168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017817.270186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335415296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017817.270202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017817.270209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86dd880 x1818247335415296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017817.270220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335415296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335415296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017817.270240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335415424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017817.270253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017817.270259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86dca80 x1818247335415424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017817.270269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335415424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335415424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017817.270308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335415552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017817.270321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017817.270327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86dd500 x1818247335415552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017817.270337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335415552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270347:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335415552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017817.270355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017817.270367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017817.270373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86de680 x1818247335415680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017817.270383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335415680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017817.270401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017817.270416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017817.270422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86dc700 x1818247335415808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017817.270432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335415808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017817.270450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017817.270461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017817.270467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017817/real 1734017817] req@ffff8800a86df100 x1818247335415936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017817.270476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335415936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017817.270484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017817.270486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017817.270489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335415936:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017817.607969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017818.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017818.609939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017819.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017819.611900:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017820.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017820.613935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017821.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017821.615940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017822.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017822.277803:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.277811:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017822.277815:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.277825:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.277838:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.277846:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017822.277882:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335416064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017822.277914:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.277919:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017822.277921:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.277926:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.277929:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.277932:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017822.277941:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017822.277953:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335416192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017822.277962:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.277966:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017822.277967:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.277970:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.277973:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.277976:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017822.277981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017822.277989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335416320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017822.277997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.278000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017822.278002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.278005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.278007:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.278011:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017822.278015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017822.278024:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335416448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017822.278027:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335416064:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017822.278033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.278036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017822.278037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.278042:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.278044:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.278048:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017822.278052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017822.278059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335416576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017822.278067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.278070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017822.278071:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.278074:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.278077:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.278080:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017822.278084:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017822.278091:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335416704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017822.278095:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017822.278099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017822.278101:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017822.278103:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017822.278106:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2086 00000100:00080000:2.0:1734017822.278109:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017822.278112:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017822.278116:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017822.278118:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86dc380 x1818247335416064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017822.278124:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335416832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017822.278145:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335416064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278155:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278158:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278163:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335416064:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017822.278169:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335416192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017822.278186:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017822.278192:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86ddf80 x1818247335416192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017822.278203:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335416192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278211:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278213:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278217:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335416192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017822.278222:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335416320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017822.278236:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017822.278242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86dfb80 x1818247335416320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017822.278252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335416320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335416320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017822.278291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335416448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017822.278304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017822.278310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86df800 x1818247335416448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017822.278321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335416448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335416448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017822.278339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335416576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017822.278350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017822.278356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86ddc00 x1818247335416576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017822.278366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335416576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335416576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017822.278385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335416704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017822.278399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017822.278405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86dc000 x1818247335416704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017822.278416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335416704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335416704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017822.278433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335416832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017822.278444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017822.278450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017822/real 1734017822] req@ffff8800a86dd180 x1818247335416832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017822.278460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335416832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017822.278467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017822.278469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017822.278472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335416832:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017822.617934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017823.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017823.619951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017824.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017824.621923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017825.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017825.623934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017826.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017826.625953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017827.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017827.285828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.285836:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017827.285841:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.285850:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.285860:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017827.285867:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017827.285901:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335416960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017827.285935:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.285939:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017827.285941:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.285946:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.285949:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017827.285952:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017827.285962:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017827.285972:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335417088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017827.285982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.285985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017827.285988:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.285992:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.285994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017827.285998:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017827.286002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017827.286010:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335417216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017827.286018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.286021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017827.286023:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.286026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.286029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017827.286032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017827.286036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017827.286046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335417344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017827.286048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335416960:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017827.286055:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.286058:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017827.286059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.286063:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.286065:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017827.286068:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017827.286072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017827.286080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335417472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017827.286089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.286093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017827.286094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.286097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.286100:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017827.286104:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017827.286107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017827.286115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017827.286115:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335417600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017827.286122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017827.286125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017827.286127:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017827.286130:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2091 00000100:00080000:2.0:1734017827.286132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017827.286136:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86de300 x1818247335416960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017827.286136:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017827.286139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017827.286146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335417728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017827.286163:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335416960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286174:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286177:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286181:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335416960:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017827.286188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017827.286205:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017827.286211:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86dce00 x1818247335417088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017827.286222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335417088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286230:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286236:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017827.286241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017827.286255:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017827.286261:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86dea00 x1818247335417216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017827.286271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335417216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017827.286308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017827.286321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017827.286327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86dd180 x1818247335417344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017827.286337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335417344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286347:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017827.286356:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335417472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017827.286367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017827.286373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86dc000 x1818247335417472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017827.286383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335417472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335417472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017827.286450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335417600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017827.286470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017827.286478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86ddc00 x1818247335417600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017827.286489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335417600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335417600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017827.286507:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335417728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017827.286519:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017827.286525:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017827/real 1734017827] req@ffff8800a86df800 x1818247335417728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017827.286534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335417728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017827.286542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017827.286544:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017827.286547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335417728:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017827.627992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017828.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017828.630022:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017829.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017829.631945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017830.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017830.633935:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017831.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017831.635959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017832.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017832.293843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.293854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017832.293859:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017832.293872:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.293883:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.293893:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017832.293940:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335417856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017832.293980:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.293984:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017832.293986:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017832.293991:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.293994:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.293998:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017832.294010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017832.294022:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335417984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017832.294032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.294035:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017832.294037:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017832.294041:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.294044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.294047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017832.294052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017832.294061:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335418112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017832.294071:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.294074:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017832.294076:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017832.294080:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.294082:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.294086:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017832.294090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017832.294101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335418240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017832.294110:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.294114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017832.294115:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017832.294117:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335417856:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017832.294120:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.294122:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.294126:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017832.294131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017832.294139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335418368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017832.294147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.294151:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017832.294153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017832.294157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.294160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.294164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017832.294168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017832.294176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335418496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017832.294184:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017832.294187:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017832.294188:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017832.294192:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017832.294192:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2096 00000100:00080000:2.0:1734017832.294196:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017832.294199:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017832.294203:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017832.294211:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335418624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017832.294215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86dfb80 x1818247335417856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335417856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335417856:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017832.294272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017832.294288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017832.294295:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86ddf80 x1818247335417984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335417984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294349:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335417984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017832.294364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335418112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017832.294382:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017832.294389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86dc380 x1818247335418112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335418112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294411:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335418112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017832.294441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335418240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017832.294455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017832.294461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86df100 x1818247335418240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335418240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335418240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017832.294491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335418368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017832.294503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017832.294509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86dc700 x1818247335418368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335418368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294526:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335418368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017832.294538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335418496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017832.294553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017832.294559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86de680 x1818247335418496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335418496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335418496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017832.294588:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335418624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017832.294600:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017832.294606:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017832/real 1734017832] req@ffff8800a86dd500 x1818247335418624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017832.294616:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335418624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017832.294623:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017832.294626:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017832.294628:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335418624:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017832.637980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017833.213770:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017833.639992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017834.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017834.641958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017835.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017835.643976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017836.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017836.645957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017837.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017837.301844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.301852:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017837.301856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.301865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00080000:2.0:1734017837.301875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.301883:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017837.301917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335418752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017837.301950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.301954:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017837.301956:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.301960:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00080000:2.0:1734017837.301964:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.301967:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017837.301976:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017837.301987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335418880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017837.301997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.302000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017837.302002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.302005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00080000:2.0:1734017837.302007:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.302010:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017837.302015:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017837.302023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335419008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017837.302031:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.302034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017837.302035:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.302040:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00080000:2.0:1734017837.302043:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.302046:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017837.302050:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017837.302059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335419136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017837.302066:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335418752:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017837.302068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.302071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017837.302073:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.302076:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00080000:2.0:1734017837.302079:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.302082:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017837.302087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017837.302095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335419264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017837.302103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.302106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017837.302107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.302111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00080000:2.0:1734017837.302113:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.302116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017837.302120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017837.302127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335419392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017837.302133:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017837.302136:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017837.302139:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017837.302141:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017837.302144:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2101 00000100:00100000:1.0:1734017837.302154:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86dca80 x1818247335418752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017837.302156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017837.302159:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017837.302163:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017837.302171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335419520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017837.302181:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335418752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302190:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302194:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302198:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335418752:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017837.302206:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335418880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017837.302222:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017837.302268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86dd880 x1818247335418880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017837.302280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335418880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335418880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017837.302301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017837.302318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017837.302324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86df480 x1818247335419008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017837.302335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335419008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017837.302373:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017837.302386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017837.302393:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86dd500 x1818247335419136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017837.302403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335419136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017837.302421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335419264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017837.302433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017837.302439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86de680 x1818247335419264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017837.302449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335419264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335419264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017837.302467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335419392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017837.302482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017837.302488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86dc700 x1818247335419392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017837.302498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335419392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335419392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017837.302515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335419520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017837.302527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017837.302533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017837/real 1734017837] req@ffff8800a86df100 x1818247335419520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017837.302542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335419520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017837.302549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017837.302551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017837.302554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335419520:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017837.647957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017838.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017838.649954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017839.213778:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017839.651985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017840.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017840.653930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017841.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017841.655948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017842.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017842.309820:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.309828:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017842.309832:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.309842:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.309851:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.309859:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017842.309893:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335419648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017842.309926:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.309930:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017842.309933:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.309937:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.309940:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.309944:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017842.309953:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017842.309963:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335419776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017842.309973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.309977:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017842.309978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.309982:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.309985:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.309988:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017842.309992:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017842.310000:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335419904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017842.310008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.310011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017842.310013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.310016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.310018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.310022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017842.310025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017842.310035:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335420032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017842.310041:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335419648:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017842.310044:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.310047:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017842.310049:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.310053:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.310055:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.310058:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017842.310062:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017842.310070:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335420160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017842.310078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.310081:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017842.310082:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.310085:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.310088:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.310091:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017842.310095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017842.310102:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335420288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017842.310108:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017842.310110:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017842.310113:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017842.310114:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017842.310118:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2106 00000100:00080000:2.0:1734017842.310121:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017842.310125:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017842.310129:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017842.310131:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86dc380 x1818247335419648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017842.310136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335420416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017842.310192:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335419648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310204:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310207:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335419648:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017842.310220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017842.310241:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017842.310248:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86ddf80 x1818247335419776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017842.310259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335419776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017842.310279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017842.310292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017842.310298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86dfb80 x1818247335419904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017842.310308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335419904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310321:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335419904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017842.310346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017842.310360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017842.310366:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86df800 x1818247335420032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017842.310376:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335420032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310384:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310386:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017842.310394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335420160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017842.310406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017842.310411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86ddc00 x1818247335420160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017842.310421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335420160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335420160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017842.310439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335420288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017842.310454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017842.310460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86dc000 x1818247335420288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017842.310470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335420288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335420288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017842.310487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335420416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017842.310499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017842.310505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017842/real 1734017842] req@ffff8800a86dd180 x1818247335420416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017842.310514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335420416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017842.310522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017842.310524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017842.310527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335420416:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017842.657928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017843.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017843.659981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017844.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017844.661931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017845.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017845.663959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017846.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017846.665953:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017847.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017847.317847:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.317856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017847.317860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.317869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2111 00000100:00080000:2.0:1734017847.317878:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.317886:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017847.317921:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335420544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017847.317954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.317958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017847.317960:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.317964:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2111 00000100:00080000:2.0:1734017847.317968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.317971:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017847.317980:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017847.317992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335420672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017847.318002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.318006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017847.318007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.318011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2111 00000100:00080000:2.0:1734017847.318014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.318017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017847.318021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017847.318029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335420800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017847.318037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.318040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017847.318042:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.318045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2111 00000100:00080000:2.0:1734017847.318048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.318051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017847.318090:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017847.318092:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335420544:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017847.318103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335420928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017847.318113:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.318116:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017847.318118:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.318122:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2111 00000100:00080000:2.0:1734017847.318125:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.318128:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017847.318133:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017847.318141:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335421056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017847.318149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.318152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017847.318154:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.318158:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2111 00000800:00000100:1.0:1734017847.318159:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017847.318161:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.318164:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017847.318168:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017847.318175:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335421184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017847.318179:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86dea00 x1818247335420544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017847.318183:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017847.318185:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017847.318187:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017847.318191:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2111 00000100:00080000:2.0:1734017847.318193:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017847.318196:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017847.318200:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017847.318205:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335420544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017847.318208:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335421312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017847.318216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318219:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318223:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335420544:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017847.318231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017847.318247:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017847.318254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86dce00 x1818247335420672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017847.318265:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335420672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017847.318272:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017847.318283:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017847.318297:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017847.318303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86de300 x1818247335420800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017847.318313:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335420800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017847.318320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017847.318351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017847.318364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017847.318370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86dd180 x1818247335420928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017847.318380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335420928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017847.318387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335420928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017847.318398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017847.318410:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017847.318416:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86dc000 x1818247335421056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017847.318426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335421056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017847.318433:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318435:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318438:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017847.318444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017847.318459:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017847.318464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86ddc00 x1818247335421184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017847.318474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335421184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017847.318482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017847.318492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017847.318504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017847.318509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017847/real 1734017847] req@ffff8800a86df800 x1818247335421312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017847.318519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335421312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017847.318526:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017847.318528:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017847.318531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421312:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017847.667928:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017848.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017848.669987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017849.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017849.671970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017850.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017850.673953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017851.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017851.675961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017852.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017852.325825:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017852.325833:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017852.325838:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017852.325848:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.325857:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.325865:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017852.325900:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335421440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017852.325933:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017852.325937:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017852.325939:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017852.325944:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.325947:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.325950:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017852.325960:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017852.326006:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335421568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017852.326018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017852.326021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017852.326024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017852.326027:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.326030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.326034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017852.326038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017852.326048:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335421696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017852.326057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017852.326060:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017852.326062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017852.326064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335421440:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017852.326066:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.326068:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.326072:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017852.326076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017852.326085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335421824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017852.326093:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017852.326096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017852.326098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017852.326101:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.326104:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.326107:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017852.326112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017852.326119:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335421952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017852.326128:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734017852.326130:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017852.326131:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017852.326133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017852.326136:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.326139:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.326142:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017852.326146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017852.326151:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86dfb80 x1818247335421440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017852.326153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335422080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017852.326160:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017852.326163:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017852.326165:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017852.326168:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2116 00000100:00080000:2.0:1734017852.326170:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017852.326173:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335421440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017852.326177:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017852.326184:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335422208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017852.326187:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326195:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335421440:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017852.326202:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335421568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017852.326219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86ddf80 x1818247335421568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017852.326236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335421568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017852.326244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326246:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335421568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017852.326272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335421696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017852.326285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86dc380 x1818247335421696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017852.326301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335421696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017852.326309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335421696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017852.326321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335421824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017852.326333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86df100 x1818247335421824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017852.326349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335421824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017852.326356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335421824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017852.326366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017852.326378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86dc700 x1818247335421952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017852.326394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335421952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017852.326401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335421952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017852.326412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017852.326427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86de680 x1818247335422080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017852.326443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335422080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017852.326450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017852.326461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017852.326473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017852.326478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017852/real 1734017852] req@ffff8800a86dd500 x1818247335422208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017852.326488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335422208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017852.326495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017852.326497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017852.326500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422208:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017852.677946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017853.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017853.680027:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017854.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017854.681990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017855.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017855.683975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017856.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017856.685959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017857.213921:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017857.333822:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.333830:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017857.333834:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.333844:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00080000:2.0:1734017857.333853:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.333860:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017857.333921:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335422336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017857.333953:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.333957:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017857.333959:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.333963:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00080000:2.0:1734017857.333972:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.333975:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017857.333986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017857.333997:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335422464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017857.334007:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.334010:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017857.334012:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.334015:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00080000:2.0:1734017857.334018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.334021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017857.334025:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017857.334033:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335422592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017857.334043:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.334046:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017857.334047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.334051:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00080000:2.0:1734017857.334053:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.334056:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017857.334060:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017857.334067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335422336:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017857.334070:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335422720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017857.334078:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.334082:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017857.334083:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.334086:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00080000:2.0:1734017857.334089:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.334092:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017857.334096:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017857.334103:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335422848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017857.334111:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.334114:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017857.334116:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.334119:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00080000:2.0:1734017857.334124:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.334127:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017857.334131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017857.334134:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017857.334139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335422976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017857.334147:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017857.334150:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017857.334151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017857.334155:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2121 00000100:00100000:1.0:1734017857.334156:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86df480 x1818247335422336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017857.334157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017857.334160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017857.334164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017857.334171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335423104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017857.334183:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335422336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334196:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334201:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335422336:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017857.334208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335422464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017857.334225:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017857.334232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86dd880 x1818247335422464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017857.334243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335422464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335422464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017857.334262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335422592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017857.334280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017857.334286:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86dca80 x1818247335422592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017857.334296:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335422592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334304:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335422592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017857.334334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335422720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017857.334346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017857.334352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86dd500 x1818247335422720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017857.334363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335422720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335422720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017857.334381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017857.334393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017857.334398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86de680 x1818247335422848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017857.334408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335422848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017857.334427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017857.334442:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017857.334448:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86dc700 x1818247335422976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017857.334459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335422976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334466:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335422976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017857.334476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335423104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017857.334488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017857.334493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017857/real 1734017857] req@ffff8800a86df100 x1818247335423104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017857.334503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335423104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017857.334511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017857.334513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017857.334516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335423104:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017857.687970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017858.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017858.689952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017859.213885:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017859.692024:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017860.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017860.694029:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017861.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017861.696014:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017862.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017862.341868:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.341878:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017862.341883:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.341895:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.341906:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.341916:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017862.341965:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335423232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017862.342003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.342007:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017862.342009:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.342014:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.342017:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.342021:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017862.342033:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017862.342044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335423360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017862.342054:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.342057:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017862.342059:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.342063:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.342066:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.342070:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017862.342074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017862.342083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335423488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017862.342091:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.342094:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017862.342096:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.342100:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.342103:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.342106:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017862.342111:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017862.342122:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335423616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017862.342130:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.342134:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017862.342135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.342139:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.342141:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.342145:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017862.342149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017862.342158:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335423744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017862.342164:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335423232:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017862.342168:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.342171:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017862.342173:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.342177:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.342179:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.342183:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017862.342188:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017862.342195:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335423872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017862.342205:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017862.342208:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017862.342210:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017862.342213:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2126 00000100:00080000:2.0:1734017862.342215:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017862.342219:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017862.342223:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017862.342231:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335424000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017862.342272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335423360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017862.342295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335423488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017862.342312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335423616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017862.342339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335423744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017862.342355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335423872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017862.342372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424000:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017862.698003:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017863.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017863.699985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017864.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017864.702021:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017865.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:0.0:1734017865.348040:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017865.348097:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017865.348124:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017865.348178:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017865.348192:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017865.348205:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017865.348216:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017865.348228:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017865.348241:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017865.348302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86ddc00 x1818247335423744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335423744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.348352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.348359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.348366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335423744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017865.348379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86dc000 x1818247335423872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335423872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.348412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.348416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.348423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335423872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017865.348434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86dd180 x1818247335424000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335424000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.348480:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.348484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.348490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017865.348501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86dc380 x1818247335423232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335423232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.348533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.348536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.348542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335423232:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017865.348552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86ddf80 x1818247335423360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348571:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335423360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.348584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.348588:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.348593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335423360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017865.348604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86dfb80 x1818247335423488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348622:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335423488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.348644:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.348649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.348654:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335423488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017865.348664:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017862/real 1734017865] req@ffff8800a86df800 x1818247335423616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017865.348682:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335423616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017865.349104:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017865.349108:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017865.349113:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335423616:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017865.703966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017866.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017866.705951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017867.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017867.349837:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.349845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017867.349850:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.349859:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.349867:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.349874:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017867.349908:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335424128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017867.349941:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.349946:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017867.349948:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.349953:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.349956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.349960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017867.349970:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017867.349980:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335424256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017867.349990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.349993:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017867.349995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.349998:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.350000:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.350003:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017867.350008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017867.350015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335424384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017867.350023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.350026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017867.350028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.350032:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.350034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.350038:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017867.350042:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017867.350051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335424512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017867.350052:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335424128:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017867.350062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.350065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017867.350067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.350070:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.350072:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.350076:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017867.350080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017867.350087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335424640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017867.350095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.350097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017867.350099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.350102:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.350113:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.350118:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017867.350119:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017867.350122:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017867.350129:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335424768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017867.350137:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017867.350139:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1734017867.350140:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86de300 x1818247335424128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017867.350141:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017867.350145:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2131 00000100:00080000:2.0:1734017867.350149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017867.350152:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017867.350156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017867.350159:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335424128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017867.350164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335424896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017867.350168:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350172:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350177:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335424128:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017867.350184:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335424256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017867.350201:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017867.350208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86dce00 x1818247335424256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017867.350220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335424256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017867.350227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335424256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017867.350238:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335424384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017867.350252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017867.350258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86dea00 x1818247335424384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017867.350268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335424384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017867.350275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335424384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017867.350304:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335424512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017867.350316:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017867.350323:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86df800 x1818247335424512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017867.350333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335424512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017867.350348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335424512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017867.350359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017867.350371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017867.350377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86dfb80 x1818247335424640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017867.350387:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335424640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017867.350394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017867.350405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017867.350416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017867.350422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86ddf80 x1818247335424768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017867.350431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335424768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017867.350439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017867.350450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017867.350462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017867.350468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017867/real 1734017867] req@ffff8800a86dc380 x1818247335424896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017867.350478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335424896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017867.350485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017867.350487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017867.350490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335424896:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017867.707971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017868.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017868.709969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017869.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017869.711955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017870.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017870.713930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017871.213779:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017871.715931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017872.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017872.357818:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017872.357826:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017872.357830:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.357840:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.357849:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017872.357856:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017872.357892:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335425024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017872.357924:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017872.357929:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017872.357931:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.357935:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.357938:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017872.357942:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017872.357951:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017872.357963:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335425152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017872.357973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017872.357976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017872.357978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.357981:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.357984:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017872.357987:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017872.357991:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017872.357999:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335425280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017872.358000:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335425024:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017872.358008:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017872.358011:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017872.358013:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.358016:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.358018:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017872.358022:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017872.358026:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017872.358035:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335425408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017872.358043:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017872.358046:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017872.358047:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.358051:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.358054:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017872.358058:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017872.358062:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017872.358069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335425536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017872.358077:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734017872.358079:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017872.358080:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017872.358081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.358085:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.358087:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017872.358090:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017872.358094:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017872.358101:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86dd180 x1818247335425024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017872.358101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335425664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017872.358109:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017872.358112:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017872.358113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017872.358116:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2136 00000100:00080000:2.0:1734017872.358119:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017872.358121:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335425024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017872.358122:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017872.358126:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734017872.358133:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1734017872.358133:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335425792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017872.358136:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358141:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335425024:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017872.358149:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335425152:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017872.358168:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017872.358175:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86dc000 x1818247335425152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017872.358194:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335425152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017872.358202:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017872.358205:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358208:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335425152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017872.358231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335425280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017872.358245:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017872.358251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86ddc00 x1818247335425280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017872.358261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335425280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017872.358269:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017872.358271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335425280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017872.358281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335425408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017872.358292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017872.358297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86df100 x1818247335425408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017872.358307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335425408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017872.358315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017872.358318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358321:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335425408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017872.358327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335425536:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017872.358340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017872.358346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86dc700 x1818247335425536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017872.358356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335425536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017872.358364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017872.358366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335425536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017872.358374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335425664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017872.358387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017872.358393:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86de680 x1818247335425664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017872.358402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335425664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017872.358410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017872.358412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335425664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017872.358420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335425792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017872.358436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017872.358442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017872/real 1734017872] req@ffff8800a86dd500 x1818247335425792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017872.358452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335425792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017872.358459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017872.358461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017872.358464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335425792:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017872.717960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017873.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017873.719957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017874.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017874.721934:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017875.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017875.723959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017876.213937:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017876.725947:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017877.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017877.365795:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.365804:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017877.365808:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.365818:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.365826:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.365833:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017877.365867:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335425920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017877.365900:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.365904:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017877.365906:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.365911:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.365914:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.365917:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017877.365927:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017877.365937:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335426048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017877.365947:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.365950:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017877.365952:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.365956:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.365959:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.365962:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017877.365966:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017877.365974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335426176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017877.365982:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.365985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017877.365987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.365990:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.365992:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.365996:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017877.365999:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017877.366008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335426304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017877.366014:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335425920:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017877.366017:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.366020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017877.366022:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.366025:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.366027:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.366031:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017877.366035:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017877.366042:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335426432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017877.366057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.366060:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017877.366061:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.366065:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.366068:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.366071:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017877.366075:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017877.366082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335426560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017877.366083:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017877.366089:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017877.366092:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017877.366094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017877.366097:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2141 00000100:00080000:2.0:1734017877.366099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017877.366102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366104:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86dca80 x1818247335425920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017877.366106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017877.366113:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335426688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017877.366123:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335425920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366134:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366137:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366142:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335425920:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017877.366149:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017877.366167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86dd880 x1818247335426048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017877.366184:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335426048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017877.366204:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017877.366218:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86df480 x1818247335426176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017877.366234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335426176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017877.366274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017877.366286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86dd500 x1818247335426304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017877.366303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335426304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017877.366322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335426432:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017877.366335:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86de680 x1818247335426432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017877.366351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335426432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335426432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017877.366369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335426560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017877.366381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86dc700 x1818247335426560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017877.366396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335426560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335426560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017877.366414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335426688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017877.366425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017877.366431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017877/real 1734017877] req@ffff8800a86df100 x1818247335426688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017877.366442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335426688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017877.366449:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017877.366451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017877.366454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335426688:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017877.727918:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017878.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017878.729957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017879.213791:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017879.731930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017880.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017880.733955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017881.213777:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017881.735951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017882.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017882.373840:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.373851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017882.373857:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017882.373869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2146 00000100:00080000:2.0:1734017882.373880:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.373890:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017882.373938:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335426816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017882.373977:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.373982:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017882.373984:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017882.373989:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2146 00000100:00080000:2.0:1734017882.373992:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.373996:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017882.374007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017882.374018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335426944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017882.374028:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.374031:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017882.374033:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017882.374038:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2146 00000100:00080000:2.0:1734017882.374041:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.374044:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017882.374049:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017882.374058:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335427072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017882.374066:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.374070:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017882.374072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017882.374076:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2146 00000100:00080000:2.0:1734017882.374079:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.374082:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017882.374086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017882.374097:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335427200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017882.374107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.374110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017882.374111:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017882.374116:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335426816:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017882.374116:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2146 00000100:00080000:2.0:1734017882.374118:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.374122:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017882.374127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017882.374137:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335427328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017882.374145:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.374149:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017882.374150:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017882.374154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2146 00000100:00080000:2.0:1734017882.374156:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.374160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017882.374164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017882.374171:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335427456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017882.374180:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017882.374184:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017882.374185:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017882.374189:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2146 00000800:00000100:1.0:1734017882.374191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017882.374192:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017882.374196:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017882.374200:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017882.374207:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335427584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017882.374214:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86ddc00 x1818247335426816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335426816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335426816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017882.374264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017882.374282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017882.374289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86dc000 x1818247335426944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335426944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335426944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017882.374322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017882.374337:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017882.374343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86dd180 x1818247335427072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335427072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374364:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374367:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017882.374392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017882.374405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017882.374411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86dc380 x1818247335427200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335427200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017882.374442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335427328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017882.374495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017882.374505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86ddf80 x1818247335427328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335427328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374530:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335427328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017882.374536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335427456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017882.374551:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017882.374558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86dfb80 x1818247335427456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374568:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335427456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374576:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335427456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017882.374587:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335427584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017882.374599:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017882.374605:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017882/real 1734017882] req@ffff8800a86df800 x1818247335427584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017882.374621:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335427584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017882.374628:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017882.374630:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017882.374633:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335427584:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017882.737988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017883.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017883.739978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017884.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017884.741969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017885.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017885.743935:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017886.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017886.745973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017887.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017887.381798:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.381808:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017887.381813:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.381825:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2151 00000100:00080000:2.0:1734017887.381835:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.381844:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017887.381890:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335427712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017887.381928:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.381932:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017887.381934:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.381939:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2151 00000100:00080000:2.0:1734017887.381942:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.381946:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017887.381959:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017887.381970:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335427840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017887.381980:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.381985:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017887.381986:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.381991:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2151 00000100:00080000:2.0:1734017887.381994:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.381997:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017887.382002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017887.382012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335427968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017887.382020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.382023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017887.382025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.382028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2151 00000100:00080000:2.0:1734017887.382031:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.382034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017887.382038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017887.382049:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335428096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017887.382059:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.382062:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1734017887.382063:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335427712:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017887.382064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.382068:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2151 00000100:00080000:2.0:1734017887.382071:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.382075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017887.382080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017887.382088:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335428224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017887.382095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.382099:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017887.382100:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.382104:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2151 00000100:00080000:2.0:1734017887.382106:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.382109:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017887.382113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017887.382121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335428352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017887.382128:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017887.382132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017887.382133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017887.382137:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2151 00000800:00000100:1.0:1734017887.382139:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017887.382140:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017887.382144:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017887.382148:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017887.382156:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335428480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017887.382162:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86dea00 x1818247335427712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382182:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335427712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382194:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382203:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335427712:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017887.382210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017887.382228:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017887.382235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86dce00 x1818247335427840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382246:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335427840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017887.382268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017887.382282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017887.382288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86de300 x1818247335427968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335427968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335427968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017887.382339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017887.382352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017887.382358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86df800 x1818247335428096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335428096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017887.382436:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335428224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017887.382456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017887.382464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86dfb80 x1818247335428224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335428224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335428224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017887.382493:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335428352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017887.382505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017887.382511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86ddf80 x1818247335428352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335428352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335428352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017887.382540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335428480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017887.382552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017887.382558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017887/real 1734017887] req@ffff8800a86dc380 x1818247335428480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017887.382574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335428480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017887.382582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017887.382584:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017887.382587:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335428480:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017887.747964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017888.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017888.750031:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017889.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017889.751970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017890.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017890.753977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017891.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017891.755975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017892.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017892.389857:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.389870:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017892.389875:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.389888:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.389899:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.389909:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017892.389959:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335428608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017892.389998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.390002:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017892.390004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.390009:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.390012:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.390016:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017892.390027:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017892.390039:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335428736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017892.390049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.390052:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017892.390054:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.390058:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.390061:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.390065:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017892.390071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017892.390080:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335428864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017892.390090:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.390093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017892.390094:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.390098:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.390100:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.390104:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017892.390108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017892.390120:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335428992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017892.390126:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335428608:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017892.390129:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.390132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017892.390134:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.390138:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.390140:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.390144:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017892.390149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017892.390157:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335429120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017892.390166:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.390169:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017892.390171:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.390175:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.390181:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.390184:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017892.390189:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017892.390196:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335429248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017892.390203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017892.390204:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017892.390207:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017892.390208:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017892.390212:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2156 00000100:00080000:2.0:1734017892.390215:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017892.390219:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017892.390223:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017892.390226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86dd180 x1818247335428608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017892.390231:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335429376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017892.390246:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335428608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335428608:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017892.390275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017892.390345:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017892.390355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86dc000 x1818247335428736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017892.390367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335428736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017892.390390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017892.390407:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017892.390413:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86ddc00 x1818247335428864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017892.390424:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335428864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390432:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390434:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017892.390463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017892.390476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017892.390482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86df100 x1818247335428992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017892.390492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335428992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335428992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017892.390512:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335429120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017892.390528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017892.390534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86dc700 x1818247335429120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017892.390545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335429120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335429120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017892.390564:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335429248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017892.390576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017892.390582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86de680 x1818247335429248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017892.390591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335429248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390599:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390601:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390604:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335429248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017892.390610:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335429376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017892.390622:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017892.390628:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017892/real 1734017892] req@ffff8800a86dd500 x1818247335429376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017892.390645:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335429376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017892.390653:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017892.390655:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017892.390658:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335429376:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017892.757929:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017893.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017893.759928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017894.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017894.761954:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017895.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017895.763951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017896.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017896.766002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017897.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017897.397830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.397839:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017897.397843:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.397853:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.397861:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017897.397869:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017897.397903:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335429504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017897.397943:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.397947:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017897.397949:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.397953:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.397956:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017897.397960:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017897.397969:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017897.397980:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335429632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017897.397989:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.397992:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017897.397994:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.397997:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.398000:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017897.398003:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017897.398007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017897.398015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335429760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017897.398023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.398026:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017897.398027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.398031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.398033:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017897.398037:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017897.398041:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017897.398051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335429888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017897.398052:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335429504:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017897.398060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.398063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017897.398065:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.398068:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.398071:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017897.398074:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017897.398079:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017897.398086:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335430016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017897.398094:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.398097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017897.398098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.398101:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.398104:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017897.398107:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017897.398111:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017897.398118:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335430144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017897.398120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017897.398126:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017897.398129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017897.398131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017897.398135:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2161 00000100:00080000:2.0:1734017897.398137:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017897.398140:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86df480 x1818247335429504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017897.398141:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017897.398145:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017897.398152:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335430272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017897.398159:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335429504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398173:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398178:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335429504:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017897.398185:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335429632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017897.398248:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017897.398259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86dd880 x1818247335429632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017897.398271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335429632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398279:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398283:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335429632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017897.398293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335429760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017897.398311:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017897.398318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86dca80 x1818247335429760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017897.398328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335429760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335429760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017897.398364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335429888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017897.398377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017897.398383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86dd500 x1818247335429888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017897.398392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335429888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335429888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017897.398411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017897.398427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017897.398433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86de680 x1818247335430016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017897.398443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335430016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017897.398461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017897.398472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017897.398478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86dc700 x1818247335430144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017897.398488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335430144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017897.398505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017897.398517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017897.398523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017897/real 1734017897] req@ffff8800a86df100 x1818247335430272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017897.398539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335430272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017897.398546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017897.398548:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017897.398551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430272:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017897.767974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017898.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017898.769954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017899.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017899.771972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017900.213922:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017900.773940:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017901.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017901.775954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017902.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017902.405859:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.405868:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017902.405872:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017902.405882:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.405891:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.405899:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017902.405935:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335430400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017902.405968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.405972:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017902.405973:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017902.405978:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.405981:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.405985:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017902.405995:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017902.406006:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335430528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017902.406016:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.406019:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017902.406021:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017902.406024:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.406026:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.406030:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017902.406034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017902.406042:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335430656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017902.406050:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.406053:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017902.406055:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017902.406058:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.406060:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.406064:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017902.406068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017902.406070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335430400:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017902.406077:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335430784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017902.406086:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.406089:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017902.406091:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017902.406095:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.406097:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.406100:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017902.406104:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017902.406111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335430912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017902.406152:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.406155:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017902.406157:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017902.406161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.406164:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.406167:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017902.406172:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017902.406174:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017902.406181:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335431040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017902.406190:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017902.406193:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017902.406195:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017902.406197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86ddc00 x1818247335430400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017902.406198:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2166 00000100:00080000:2.0:1734017902.406201:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017902.406204:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017902.406208:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017902.406215:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335430400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017902.406216:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335431168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017902.406225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335430400:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017902.406241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335430528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017902.406259:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017902.406266:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86dc000 x1818247335430528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017902.406277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335430528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017902.406285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335430528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017902.406298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335430656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017902.406311:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017902.406317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86dd180 x1818247335430656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017902.406328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335430656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017902.406335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335430656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017902.406367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335430784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017902.406379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017902.406385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86dc380 x1818247335430784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017902.406395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335430784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017902.406403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335430784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017902.406413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430912:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017902.406429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017902.406435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86ddf80 x1818247335430912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017902.406445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335430912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017902.406452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335430912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017902.406463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017902.406474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017902.406480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86dfb80 x1818247335431040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017902.406489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335431040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017902.406497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017902.406507:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017902.406519:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017902.406525:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017902/real 1734017902] req@ffff8800a86df800 x1818247335431168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017902.406542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335431168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017902.406550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017902.406552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017902.406555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431168:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017902.777955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017903.213799:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017903.779930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017904.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017904.781951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017905.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017905.783950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017906.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017906.785924:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017907.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017907.413826:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.413834:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017907.413839:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.413848:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.413857:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017907.413865:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017907.413901:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335431296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017907.413935:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.413939:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017907.413941:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.413945:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.413949:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017907.413952:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017907.413961:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017907.413972:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335431424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017907.413981:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.413984:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017907.413987:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.413991:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.413993:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017907.413996:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017907.414001:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017907.414009:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335431552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017907.414017:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.414020:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017907.414021:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.414025:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.414027:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017907.414031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335431296:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734017907.414076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017907.414089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335431680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017907.414099:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.414102:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017907.414104:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.414107:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.414110:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017907.414113:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017907.414117:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017907.414125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335431808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017907.414134:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.414137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1734017907.414139:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017907.414139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.414142:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.414145:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017907.414148:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017907.414152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017907.414160:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86de300 x1818247335431296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017907.414160:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335431936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017907.414168:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017907.414171:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017907.414172:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017907.414175:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2171 00000100:00080000:2.0:1734017907.414178:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017907.414179:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335431296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017907.414181:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017907.414185:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734017907.414189:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414192:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017907.414193:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335432064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017907.414197:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335431296:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017907.414204:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335431424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017907.414221:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414228:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86dce00 x1818247335431424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017907.414239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335431424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017907.414247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017907.414254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335431424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017907.414260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335431552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017907.414273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414279:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86dea00 x1818247335431552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017907.414289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335431552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017907.414296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017907.414302:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335431552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017907.414326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335431680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017907.414338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86df800 x1818247335431680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017907.414355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335431680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017907.414363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017907.414368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335431680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017907.414373:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017907.414389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86dfb80 x1818247335431808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017907.414405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335431808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017907.414412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017907.414417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017907.414423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017907.414434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86ddf80 x1818247335431936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017907.414450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335431936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017907.414458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017907.414463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335431936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017907.414468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017907.414480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017907.414485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017907/real 1734017907] req@ffff8800a86dc380 x1818247335432064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017907.414501:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335432064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017907.414508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017907.414510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017907.414513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432064:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017907.787913:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017908.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017908.789934:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017909.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017909.791986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017910.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017910.793931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017911.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017911.795948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017912.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017912.421823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017912.421832:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017912.421836:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017912.421846:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.421854:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017912.421861:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017912.421896:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335432192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017912.421929:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017912.421933:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017912.421935:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017912.421939:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.421943:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017912.421946:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017912.422000:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017912.422013:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335432320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017912.422024:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017912.422027:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017912.422029:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017912.422033:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.422036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017912.422040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017912.422044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:1.0:1734017912.422045:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335432192:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017912.422052:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335432448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017912.422062:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017912.422065:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017912.422067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017912.422070:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.422073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017912.422076:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017912.422080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017912.422089:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335432576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017912.422097:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017912.422100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017912.422101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017912.422105:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.422108:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734017912.422111:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017912.422111:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017912.422115:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017912.422123:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335432704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017912.422131:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017912.422132:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86dd180 x1818247335432192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017912.422133:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017912.422135:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017912.422138:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.422142:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017912.422145:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017912.422149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017912.422151:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335432192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017912.422157:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335432832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017912.422162:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422165:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734017912.422165:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017912.422168:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017912.422169:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017912.422170:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335432192:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017912.422172:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2176 00000100:00080000:2.0:1734017912.422175:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017912.422177:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335432320:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734017912.422178:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017912.422182:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017912.422189:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335432960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017912.422194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017912.422201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86dc000 x1818247335432320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017912.422212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335432320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017912.422220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017912.422226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335432320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017912.422248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335432448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017912.422263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017912.422269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86ddc00 x1818247335432448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017912.422280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335432448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017912.422287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017912.422293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335432448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017912.422298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335432576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017912.422309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017912.422315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86df100 x1818247335432576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017912.422325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335432576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017912.422332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422334:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017912.422337:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335432576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017912.422343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432704:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017912.422359:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017912.422365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86dc700 x1818247335432704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017912.422375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335432704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017912.422383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017912.422388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017912.422393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017912.422405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017912.422411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86de680 x1818247335432832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017912.422421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335432832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017912.422429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017912.422434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017912.422439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017912.422451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017912.422457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017912/real 1734017912] req@ffff8800a86dd500 x1818247335432960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017912.422474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335432960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017912.422481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017912.422483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017912.422486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335432960:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017912.797984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017913.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017913.799995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017914.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017914.801986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017915.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017915.803970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017916.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017916.805953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017917.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017917.429830:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.429838:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017917.429842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.429852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.429895:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.429905:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017917.429939:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335433088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017917.429971:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.429975:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017917.429977:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.429981:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.429985:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.429988:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017917.429997:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017917.430008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335433216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017917.430018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.430021:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017917.430023:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.430026:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.430028:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.430032:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017917.430036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017917.430044:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335433344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017917.430052:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.430055:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017917.430057:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.430061:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.430063:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.430067:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017917.430072:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017917.430082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335433088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017917.430082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335433472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017917.430090:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.430094:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017917.430095:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.430099:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.430101:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.430105:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017917.430108:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017917.430116:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335433600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017917.430124:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.430127:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017917.430128:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.430131:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.430134:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.430137:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017917.430141:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017917.430147:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335433728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017917.430150:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017917.430156:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017917.430159:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017917.430161:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017917.430164:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2181 00000100:00080000:2.0:1734017917.430167:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017917.430170:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430171:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86dca80 x1818247335433088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017917.430174:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017917.430181:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335433856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017917.430189:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335433088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430200:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430203:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430208:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335433088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017917.430216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335433216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017917.430232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86dd880 x1818247335433216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017917.430249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335433216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430257:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335433216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017917.430270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335433344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017917.430284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86df480 x1818247335433344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017917.430299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335433344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335433344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017917.430336:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335433472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017917.430348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86dd500 x1818247335433472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017917.430365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335433472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335433472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017917.430383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335433600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017917.430398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86de680 x1818247335433600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017917.430414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335433600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335433600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017917.430432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335433728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017917.430443:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86dc700 x1818247335433728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017917.430459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335433728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335433728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017917.430478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335433856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017917.430489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017917.430495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017917/real 1734017917] req@ffff8800a86df100 x1818247335433856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017917.430512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335433856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017917.430519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017917.430521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017917.430524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335433856:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017917.808038:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017918.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017918.809997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017919.213875:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017919.811922:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017920.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017920.813977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017921.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017921.816009:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017922.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017922.437863:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.437873:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017922.437878:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.437890:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.437902:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.437911:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017922.437960:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335433984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017922.437997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.438002:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017922.438004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.438009:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.438013:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.438016:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017922.438028:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017922.438040:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335434112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017922.438050:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.438054:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017922.438056:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.438060:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.438062:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.438065:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017922.438071:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017922.438079:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335434240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017922.438088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.438091:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017922.438093:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.438096:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.438099:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.438102:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017922.438106:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017922.438117:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335434368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017922.438122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335433984:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017922.438128:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.438132:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017922.438133:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.438137:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.438140:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.438143:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017922.438147:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017922.438155:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335434496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017922.438163:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.438166:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017922.438168:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.438171:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.438174:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.438178:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017922.438182:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017922.438190:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335434624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017922.438197:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017922.438198:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017922.438201:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017922.438202:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017922.438207:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2186 00000100:00080000:2.0:1734017922.438210:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017922.438214:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017922.438218:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017922.438221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86ddc00 x1818247335433984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017922.438226:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335434752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017922.438241:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335433984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335433984:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017922.438271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335434112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017922.438289:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017922.438296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86dc000 x1818247335434112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017922.438307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335434112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335434112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017922.438329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335434240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017922.438344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017922.438350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86dd180 x1818247335434240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017922.438360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335434240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335434240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017922.438400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335434368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017922.438413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017922.438419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86dc380 x1818247335434368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017922.438429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335434368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335434368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017922.438449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335434496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017922.438465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017922.438471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86ddf80 x1818247335434496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017922.438482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335434496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335434496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017922.438501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335434624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017922.438513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017922.438518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86dfb80 x1818247335434624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017922.438528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335434624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335434624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017922.438547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335434752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017922.438559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017922.438565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017922/real 1734017922] req@ffff8800a86df800 x1818247335434752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017922.438581:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335434752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017922.438588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017922.438591:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017922.438593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335434752:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017922.817991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017923.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017923.820002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017924.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017924.822000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017925.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017925.823913:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017926.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017926.825927:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017927.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017927.445799:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.445808:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017927.445812:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.445822:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00080000:2.0:1734017927.445831:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.445838:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017927.445875:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335434880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017927.445910:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.445915:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017927.445917:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.445921:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00080000:2.0:1734017927.445924:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.445927:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017927.445937:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017927.445948:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335435008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017927.445958:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.445962:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017927.445963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.445967:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00080000:2.0:1734017927.445969:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.445972:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017927.445976:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017927.445984:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335435136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017927.445993:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.445996:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017927.445997:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.446001:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00080000:2.0:1734017927.446003:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.446006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017927.446010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017927.446020:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335435264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017927.446029:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.446032:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017927.446033:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.446038:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00100000:1.0:1734017927.446040:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335434880:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017927.446040:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.446044:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017927.446048:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017927.446055:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335435392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017927.446063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.446066:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017927.446067:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.446070:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00080000:2.0:1734017927.446073:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.446076:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017927.446080:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017927.446087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335435520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017927.446095:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017927.446098:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017927.446099:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017927.446103:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2191 00000100:00080000:2.0:1734017927.446105:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017927.446108:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017927.446113:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017927.446121:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335435648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017927.446132:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017927.446154:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017927.446178:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017927.446193:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335435392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017927.446208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335435520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017927.446221:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335435648:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017927.827946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017928.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017928.829980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017929.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017929.831969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017930.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:0.0:1734017930.451969:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017930.452016:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017930.452034:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017930.452066:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017930.452075:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017930.452082:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017930.452088:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017930.452094:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017930.452100:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017930.452178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86df800 x1818247335435264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335435264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452239:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017930.452259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86dfb80 x1818247335435392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335435392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335435392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017930.452314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86ddf80 x1818247335435520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335435520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335435520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017930.452379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86dc380 x1818247335435648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335435648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335435648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017930.452432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86dea00 x1818247335434880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335434880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335434880:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017930.452484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86dce00 x1818247335435008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335435008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452525:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017930.452535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017927/real 1734017930] req@ffff8800a86de300 x1818247335435136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017930.452553:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335435136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017930.452565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017930.452569:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017930.452574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435136:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017930.833936:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017931.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017931.835922:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017932.213927:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017932.453814:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017932.453823:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017932.453827:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017932.453837:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.453845:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.453852:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017932.453887:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335435776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017932.453920:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017932.453924:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017932.453925:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017932.453930:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.453933:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.453937:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017932.453946:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017932.453957:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335435904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017932.453966:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017932.453969:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017932.453971:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017932.453975:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.453978:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.453981:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017932.453985:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017932.453993:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335436032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017932.454001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017932.454004:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017932.454006:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017932.454009:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.454011:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.454015:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017932.454019:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017932.454028:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335436160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017932.454036:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017932.454037:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335435776:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017932.454039:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017932.454041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017932.454044:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.454047:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.454050:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017932.454054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017932.454061:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335436288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017932.454070:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017932.454073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017932.454074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017932.454077:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.454080:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.454084:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017932.454087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017932.454094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335436416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017932.454102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017932.454105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017932.454106:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017932.454108:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017932.454110:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2196 00000100:00080000:2.0:1734017932.454112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017932.454115:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017932.454119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017932.454126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335436544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017932.454128:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86dd180 x1818247335435776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454147:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335435776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454158:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454162:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454166:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335435776:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017932.454173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017932.454189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017932.454196:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86dc000 x1818247335435904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454208:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335435904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454215:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454218:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335435904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017932.454227:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017932.454244:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017932.454250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86ddc00 x1818247335436032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454260:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335436032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454268:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017932.454298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017932.454310:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017932.454316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86df100 x1818247335436160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335436160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454336:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017932.454345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335436288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017932.454356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017932.454362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86dc700 x1818247335436288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335436288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454381:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335436288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017932.454401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335436416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017932.454417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017932.454423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86de680 x1818247335436416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335436416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335436416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017932.454450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335436544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017932.454461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017932.454467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017932/real 1734017932] req@ffff8800a86dd500 x1818247335436544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017932.454477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335436544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017932.454484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017932.454486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017932.454489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335436544:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017932.838002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017933.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017933.839958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017934.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017934.841949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017935.213793:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017935.843972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017936.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017936.845958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017937.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017937.461818:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.461827:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017937.461833:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017937.461845:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2201 00000100:00080000:2.0:1734017937.461856:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.461866:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017937.461911:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335436672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017937.461950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.461954:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017937.461956:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017937.461961:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2201 00000100:00080000:2.0:1734017937.461965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.461968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017937.461979:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017937.461991:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335436800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017937.462001:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.462005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017937.462007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017937.462011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2201 00000100:00080000:2.0:1734017937.462014:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.462017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017937.462021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017937.462030:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335436928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017937.462038:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.462041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017937.462044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017937.462047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2201 00000100:00080000:2.0:1734017937.462050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.462053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017937.462057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017937.462067:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335437056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017937.462076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.462079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017937.462081:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017937.462082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335436672:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017937.462085:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2201 00000100:00080000:2.0:1734017937.462087:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.462091:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017937.462095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017937.462104:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335437184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017937.462112:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.462116:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017937.462118:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017937.462121:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2201 00000100:00080000:2.0:1734017937.462124:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.462127:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017937.462131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017937.462139:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335437312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017937.462148:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017937.462151:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017937.462153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017937.462156:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2201 00000800:00000100:1.0:1734017937.462158:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017937.462160:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017937.462163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017937.462167:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017937.462175:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335437440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017937.462183:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86df480 x1818247335436672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462203:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335436672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462215:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462219:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462224:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335436672:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017937.462232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017937.462249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017937.462256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86dd880 x1818247335436800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335436800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017937.462287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017937.462301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017937.462307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86dca80 x1818247335436928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335436928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335436928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017937.462358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017937.462371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017937.462377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86dd500 x1818247335437056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335437056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017937.462407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335437184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017937.462419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017937.462425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86de680 x1818247335437184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335437184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335437184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017937.462454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335437312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017937.462470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017937.462476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86dc700 x1818247335437312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335437312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335437312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017937.462505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335437440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017937.462517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017937.462561:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017937/real 1734017937] req@ffff8800a86df100 x1818247335437440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017937.462573:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335437440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017937.462581:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017937.462583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017937.462586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335437440:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017937.848007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017938.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017938.849956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017939.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017939.851832:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017940.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017940.853894:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017941.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017941.855965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017942.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017942.469844:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.469853:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017942.469858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.469868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.469877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.469884:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017942.469921:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335437568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017942.469954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.469958:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017942.469960:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.469965:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.469968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.469971:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017942.469981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017942.469992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335437696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017942.470002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.470005:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017942.470007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.470010:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.470013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.470016:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017942.470020:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017942.470029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335437824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017942.470037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.470040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017942.470041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.470045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.470047:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.470051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017942.470054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017942.470064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335437952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017942.470071:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335437568:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017942.470073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.470076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017942.470078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.470083:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.470085:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.470088:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017942.470093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017942.470100:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335438080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017942.470108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.470111:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017942.470112:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.470115:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.470118:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.470121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017942.470125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017942.470132:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335438208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017942.470139:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017942.470140:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017942.470143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017942.470144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017942.470147:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2207 00000100:00080000:2.0:1734017942.470150:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017942.470153:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017942.470157:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017942.470160:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86ddc00 x1818247335437568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017942.470165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335438336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017942.470186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335437568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470196:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470199:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470204:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335437568:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017942.470211:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017942.470227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017942.470233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86dc000 x1818247335437696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017942.470244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335437696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017942.470264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017942.470283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017942.470289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86dd180 x1818247335437824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017942.470299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335437824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017942.470338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017942.470350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017942.470356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86de300 x1818247335437952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017942.470367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335437952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470374:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335437952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017942.470385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017942.470397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017942.470403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86dce00 x1818247335438080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017942.470413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335438080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470420:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017942.470431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017942.470493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017942.470501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86dea00 x1818247335438208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017942.470512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335438208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017942.470531:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017942.470544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017942.470550:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017942/real 1734017942] req@ffff8800a86dc380 x1818247335438336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017942.470560:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335438336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017942.470568:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017942.470570:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017942.470573:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438336:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017942.857990:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017943.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017943.859991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017944.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017944.861971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017945.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017945.863958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017946.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017946.865939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017947.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017947.477833:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.477842:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017947.477847:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017947.477857:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.477865:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.477872:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017947.477909:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335438464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017947.477941:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.477945:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017947.477947:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017947.477951:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.477954:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.477958:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017947.477967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017947.477977:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335438592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017947.477987:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.477990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017947.477992:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017947.477996:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.477998:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.478002:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017947.478007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017947.478015:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335438720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017947.478023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.478027:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017947.478028:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017947.478031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.478034:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.478037:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017947.478041:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017947.478047:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335438464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017947.478051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335438848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017947.478060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.478063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017947.478064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017947.478067:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.478070:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.478073:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017947.478077:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017947.478085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335438976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017947.478093:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.478096:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017947.478098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017947.478102:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.478105:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.478108:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017947.478112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734017947.478113:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734017947.478119:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335439104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017947.478127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017947.478129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017947.478131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734017947.478134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86ddf80 x1818247335438464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017947.478134:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2212 00000100:00080000:2.0:1734017947.478137:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017947.478140:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017947.478144:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017947.478151:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335439232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017947.478161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335438464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478172:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478175:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335438464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017947.478187:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335438592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017947.478203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017947.478210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86dfb80 x1818247335438592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017947.478221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335438592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335438592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017947.478240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335438720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017947.478253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017947.478259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86df800 x1818247335438720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017947.478269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335438720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335438720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017947.478306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335438848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017947.478318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017947.478324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86dc380 x1818247335438848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017947.478334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335438848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478344:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335438848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017947.478397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017947.478414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017947.478421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86dea00 x1818247335438976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017947.478431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335438976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335438976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017947.478449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335439104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017947.478464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017947.478471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86dce00 x1818247335439104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017947.478481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335439104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335439104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017947.478499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335439232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017947.478511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017947.478517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017947/real 1734017947] req@ffff8800a86de300 x1818247335439232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017947.478526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335439232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017947.478534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017947.478536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017947.478539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335439232:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017947.867936:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734017948.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017948.869939:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017949.213962:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017949.871972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017950.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017950.873965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017951.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017951.875946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017952.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017952.485841:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017952.485850:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017952.485854:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.485864:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.485873:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.485881:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017952.485918:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335439360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017952.485952:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017952.485956:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017952.485958:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.485962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.485965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.485969:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017952.485978:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017952.485989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335439488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017952.485998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017952.486002:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017952.486003:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.486007:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.486009:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.486013:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017952.486017:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017952.486025:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335439616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017952.486033:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017952.486036:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017952.486038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.486042:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.486044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.486047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017952.486052:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017952.486060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335439744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017952.486069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017952.486072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335439360:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017952.486072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017952.486074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.486078:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.486080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.486084:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017952.486087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017952.486095:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335439872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017952.486102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017952.486105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017952.486107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.486110:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.486112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.486116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017952.486120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017952.486126:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335440000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017952.486135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017952.486138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1734017952.486139:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017952.486139:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017952.486143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2217 00000100:00080000:2.0:1734017952.486146:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017952.486149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017952.486152:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017952.486159:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335440128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017952.486161:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86dd180 x1818247335439360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335439360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486197:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486201:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486205:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335439360:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017952.486212:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335439488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017952.486229:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017952.486236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86dc000 x1818247335439488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486246:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335439488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486260:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335439488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017952.486296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335439616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017952.486314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017952.486321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86ddc00 x1818247335439616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335439616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335439616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017952.486373:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335439744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017952.486386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017952.486392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86df100 x1818247335439744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335439744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335439744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017952.486421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335439872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017952.486432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017952.486438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86dc700 x1818247335439872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335439872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335439872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017952.486465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017952.486480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017952.486486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86de680 x1818247335440000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335440000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017952.486514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017952.486525:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017952.486531:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017952/real 1734017952] req@ffff8800a86dd500 x1818247335440128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017952.486541:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335440128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017952.486548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017952.486550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017952.486553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440128:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017952.877928:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017953.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017953.879955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017954.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017954.881957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017955.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017955.883935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017956.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017956.885825:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017957.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017957.493843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017957.493851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017957.493855:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.493865:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.493874:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.493882:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017957.493917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335440256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017957.493950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017957.493954:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017957.493956:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.493961:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.493964:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.493967:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017957.493977:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017957.493987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335440384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017957.493997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017957.494000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017957.494002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.494005:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.494008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.494011:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017957.494016:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017957.494023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335440512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017957.494032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017957.494035:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017957.494036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.494040:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.494042:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.494045:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017957.494049:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017957.494056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335440256:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734017957.494059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335440640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017957.494068:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017957.494071:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017957.494072:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.494076:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.494078:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.494082:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017957.494086:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017957.494093:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335440768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017957.494101:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017957.494104:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017957.494105:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.494109:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.494120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.494124:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734017957.494125:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734017957.494128:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017957.494135:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335440896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017957.494143:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017957.494146:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86dca80 x1818247335440256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017957.494146:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017957.494147:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017957.494150:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2222 00000100:00080000:2.0:1734017957.494154:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017957.494157:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017957.494161:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017957.494169:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335441024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017957.494172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335440256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494223:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335440256:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017957.494231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335440384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017957.494251:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017957.494259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86dd880 x1818247335440384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017957.494271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335440384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335440384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017957.494290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335440512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017957.494304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017957.494310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86df480 x1818247335440512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017957.494320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335440512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335440512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017957.494358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335440640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017957.494371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017957.494378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86dd500 x1818247335440640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017957.494388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335440640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335440640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017957.494406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017957.494417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017957.494423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86de680 x1818247335440768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017957.494433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335440768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017957.494451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017957.494466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017957.494472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86dc700 x1818247335440896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017957.494483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335440896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335440896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017957.494500:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017957.494511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017957.494517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017957/real 1734017957] req@ffff8800a86df100 x1818247335441024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017957.494527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335441024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017957.494535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017957.494537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017957.494540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441024:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017957.887954:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017958.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017958.890000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017959.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017959.891990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017960.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017960.893992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017961.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017961.895982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017962.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017962.501861:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.501872:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017962.501878:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.501891:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.501903:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.501912:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017962.501961:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335441152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017962.502003:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.502007:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017962.502009:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.502014:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.502018:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.502022:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017962.502033:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017962.502046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335441280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017962.502056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.502060:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017962.502062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.502065:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.502068:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.502072:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017962.502077:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017962.502085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335441408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017962.502125:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.502129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017962.502131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.502135:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.502138:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.502142:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017962.502146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017962.502159:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335441536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017962.502167:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335441152:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017962.502169:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.502173:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017962.502174:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.502180:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.502182:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.502186:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017962.502191:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017962.502199:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335441664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017962.502207:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.502211:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017962.502212:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.502216:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.502220:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.502223:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017962.502227:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017962.502235:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335441792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017962.502241:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017962.502243:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017962.502246:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017962.502247:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017962.502251:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2227 00000100:00080000:2.0:1734017962.502254:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017962.502258:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017962.502262:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017962.502265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86ddc00 x1818247335441152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017962.502270:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335441920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017962.502292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335441152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335441152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017962.502320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335441280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017962.502338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017962.502345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86dc000 x1818247335441280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017962.502356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335441280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502371:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335441280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017962.502378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335441408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017962.502392:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017962.502398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86dd180 x1818247335441408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017962.502408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335441408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335441408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017962.502448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335441536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017962.502462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017962.502468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86de300 x1818247335441536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017962.502479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335441536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335441536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017962.502497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017962.502510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017962.502516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86dce00 x1818247335441664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017962.502526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335441664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017962.502545:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017962.502560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017962.502567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86dea00 x1818247335441792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017962.502577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335441792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502587:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502590:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017962.502595:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017962.502608:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017962.502613:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017962/real 1734017962] req@ffff8800a86dc380 x1818247335441920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017962.502623:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335441920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017962.502630:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017962.502632:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017962.502636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335441920:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017962.897937:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017963.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017963.900004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017964.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017964.901949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017965.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017965.903966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017966.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017966.905931:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017967.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017967.509823:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017967.509831:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017967.509835:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.509845:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2232 00000100:00080000:2.0:1734017967.509854:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017967.509862:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017967.509898:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335442048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017967.509934:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017967.509940:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017967.509942:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.509949:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2232 00000100:00080000:2.0:1734017967.509954:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017967.509959:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017967.509973:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017967.509991:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335442176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017967.510006:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017967.510008:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335442048:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017967.510041:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017967.510045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.510051:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2232 00000100:00080000:2.0:1734017967.510056:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017967.510061:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017967.510068:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017967.510083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335442304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017967.510096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017967.510100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017967.510103:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.510108:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2232 00000800:00000100:1.0:1734017967.510111:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017967.510111:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017967.510117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017967.510123:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017967.510135:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86df800 x1818247335442048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017967.510136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335442432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017967.510150:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017967.510155:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017967.510158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.510163:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2232 00000100:00080000:2.0:1734017967.510166:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734017967.510170:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335442048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017967.510171:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017967.510177:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1734017967.510185:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017967.510189:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734017967.510189:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335442560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017967.510196:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335442048:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017967.510203:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017967.510207:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017967.510209:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.510214:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2232 00000100:00080000:2.0:1734017967.510217:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017967.510222:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510224:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335442176:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734017967.510228:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017967.510240:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335442688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017967.510250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017967.510255:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017967.510260:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1734017967.510261:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86dfb80 x1818247335442176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734017967.510262:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017967.510267:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2232 00000100:00080000:2.0:1734017967.510272:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017967.510276:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335442176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017967.510282:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734017967.510291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1734017967.510294:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335442816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:1.0:1734017967.510295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017967.510300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335442176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017967.510310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335442304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017967.510334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86ddf80 x1818247335442304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017967.510364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335442304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017967.510377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017967.510386:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017967.510392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335442304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017967.510401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335442432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017967.510424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86dc380 x1818247335442432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017967.510442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335442432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017967.510450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017967.510452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017967.510455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335442432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017967.510461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335442560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017967.510473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86dea00 x1818247335442560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017967.510489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335442560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017967.510496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017967.510498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017967.510501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335442560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017967.510520:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335442688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017967.510535:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86dce00 x1818247335442688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017967.510552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335442688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017967.510559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017967.510562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017967.510565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335442688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017967.510570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335442816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017967.510581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017967.510587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017967/real 1734017967] req@ffff8800a86de300 x1818247335442816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017967.510596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335442816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017967.510603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017967.510606:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017967.510609:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335442816:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017967.907951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017968.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017968.909956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017969.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017969.912032:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017970.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017970.913910:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017971.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017971.915970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017972.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017972.517835:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017972.517844:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017972.517848:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.517858:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.517867:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.517874:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017972.517910:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335442944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017972.517972:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017972.517976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017972.517978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.517983:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.517986:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.517990:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017972.518000:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017972.518012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335443072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017972.518022:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017972.518025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017972.518027:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.518031:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.518033:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.518036:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017972.518040:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017972.518048:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335443200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017972.518058:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017972.518061:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017972.518063:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.518066:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.518068:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.518072:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017972.518076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017972.518085:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335443328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017972.518094:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017972.518095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335442944:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017972.518097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017972.518098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.518102:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.518104:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.518107:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017972.518111:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017972.518118:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335443456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017972.518127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017972.518130:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017972.518131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.518135:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.518138:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.518141:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017972.518146:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017972.518153:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335443584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017972.518161:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734017972.518163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017972.518163:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017972.518165:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017972.518168:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2237 00000100:00080000:2.0:1734017972.518171:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017972.518174:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017972.518178:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017972.518184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86dd180 x1818247335442944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017972.518184:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335443712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017972.518211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335442944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518222:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518225:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518230:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335442944:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017972.518237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017972.518253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017972.518260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86dc000 x1818247335443072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017972.518271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335443072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518279:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017972.518290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017972.518303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017972.518309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86ddc00 x1818247335443200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017972.518319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335443200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017972.518364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017972.518376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017972.518383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86df100 x1818247335443328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017972.518393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335443328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017972.518411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335443456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017972.518423:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017972.518429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86dc700 x1818247335443456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017972.518438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335443456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335443456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017972.518457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335443584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017972.518471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017972.518478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86de680 x1818247335443584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017972.518488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335443584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335443584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017972.518505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335443712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017972.518516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017972.518522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017972/real 1734017972] req@ffff8800a86dd500 x1818247335443712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017972.518532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335443712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734017997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017972.518539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017972.518541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017972.518544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335443712:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017972.917969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017973.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017973.919970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017974.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017974.921961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017975.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017975.923960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017976.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017976.925998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017977.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017977.525824:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.525833:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017977.525837:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.525881:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.525890:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.525897:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017977.525932:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335443840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017977.525964:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.525968:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017977.525970:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.525974:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.525977:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.525981:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017977.525990:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017977.526001:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335443968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017977.526011:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.526014:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017977.526015:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.526019:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.526021:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.526025:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017977.526029:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017977.526037:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335444096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017977.526045:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.526048:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017977.526049:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.526053:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.526056:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.526059:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017977.526063:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017977.526073:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335444224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017977.526076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335443840:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017977.526082:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.526085:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017977.526087:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.526090:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.526092:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.526095:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017977.526099:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017977.526107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335444352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017977.526115:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.526118:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017977.526120:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.526123:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.526126:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.526129:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017977.526133:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017977.526140:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335444480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734017977.526143:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017977.526149:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017977.526152:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017977.526153:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017977.526157:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2242 00000100:00080000:2.0:1734017977.526159:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017977.526163:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526164:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86df480 x1818247335443840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734017977.526166:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017977.526173:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335444608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017977.526190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335443840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526201:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526208:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335443840:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017977.526216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017977.526231:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86dd880 x1818247335443968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017977.526249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335443968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526257:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335443968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017977.526268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017977.526281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86dca80 x1818247335444096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017977.526297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335444096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017977.526335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017977.526348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86dd500 x1818247335444224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017977.526365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335444224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017977.526384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335444352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017977.526395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86de680 x1818247335444352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017977.526411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335444352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335444352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017977.526429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335444480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017977.526444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86dc700 x1818247335444480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017977.526459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335444480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335444480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017977.526477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335444608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017977.526488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017977.526494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017977/real 1734017977] req@ffff8800a86df100 x1818247335444608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017977.526504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335444608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017977.526511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017977.526513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017977.526516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335444608:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017977.927999:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017978.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017978.929979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017979.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017979.932003:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017980.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017980.934008:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017981.213882:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017981.936003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017982.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017982.533850:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017982.533858:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017982.533862:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.533872:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.533882:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.533889:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017982.533924:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335444736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017982.533956:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017982.533960:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017982.533963:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.533968:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.533971:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.533975:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017982.533984:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017982.533994:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335444864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017982.534004:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017982.534008:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017982.534010:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.534013:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.534016:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.534019:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017982.534024:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017982.534032:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335444992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017982.534040:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017982.534043:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017982.534044:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.534047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.534050:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.534053:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017982.534057:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017982.534066:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335445120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017982.534074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335444736:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017982.534076:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017982.534079:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017982.534080:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.534084:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.534086:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.534090:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017982.534093:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017982.534101:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335445248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017982.534108:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017982.534111:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017982.534113:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.534116:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.534120:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.534123:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017982.534127:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017982.534134:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335445376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017982.534141:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1734017982.534142:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017982.534144:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017982.534146:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017982.534149:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2247 00000100:00080000:2.0:1734017982.534153:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017982.534156:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017982.534160:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734017982.534164:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86ddc00 x1818247335444736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734017982.534167:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335445504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017982.534191:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335444736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534200:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534208:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335444736:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017982.534215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017982.534232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017982.534239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86dc000 x1818247335444864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017982.534250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335444864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017982.534269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017982.534282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017982.534288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86dd180 x1818247335444992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017982.534298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335444992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335444992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017982.534335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335445120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017982.534348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017982.534354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86de300 x1818247335445120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017982.534365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335445120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335445120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017982.534382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335445248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017982.534394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017982.534400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86dce00 x1818247335445248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017982.534410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335445248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534417:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335445248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017982.534428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335445376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017982.534443:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017982.534449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86dea00 x1818247335445376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017982.534460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335445376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335445376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017982.534477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335445504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017982.534488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017982.534495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017982/real 1734017982] req@ffff8800a86dc380 x1818247335445504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017982.534504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335445504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017982.534512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017982.534514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017982.534517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335445504:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017982.937976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017983.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017983.940003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017984.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017984.941992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017985.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017985.943961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017986.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017986.945993:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017987.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017987.541845:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017987.541854:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017987.541858:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017987.541868:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.541877:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.541885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017987.541922:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335445632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017987.541954:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017987.541959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017987.541961:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017987.541965:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.541968:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.541972:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017987.541981:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017987.541992:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335445760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017987.542002:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017987.542006:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017987.542007:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017987.542011:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.542013:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.542017:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017987.542021:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017987.542029:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335445888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017987.542037:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017987.542040:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017987.542041:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017987.542045:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.542047:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.542050:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017987.542054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017987.542064:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335446016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017987.542073:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017987.542074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335445632:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017987.542076:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017987.542078:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017987.542082:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.542084:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.542087:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017987.542092:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017987.542099:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335446144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017987.542107:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017987.542110:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017987.542111:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017987.542115:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.542117:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.542121:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017987.542124:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017987.542132:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335446272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017987.542139:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017987.542142:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017987.542143:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734017987.542144:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017987.542147:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2252 00000100:00080000:2.0:1734017987.542149:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017987.542152:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017987.542156:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017987.542164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335446400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017987.542166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86ddf80 x1818247335445632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542193:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335445632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542211:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335445632:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017987.542218:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335445760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017987.542234:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017987.542241:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86dfb80 x1818247335445760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335445760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335445760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017987.542272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335445888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017987.542284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017987.542290:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86df800 x1818247335445888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335445888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335445888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017987.542339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017987.542351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017987.542357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86dc380 x1818247335446016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335446016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017987.542386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335446144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017987.542397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017987.542403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86dea00 x1818247335446144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335446144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542420:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335446144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017987.542431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335446272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017987.542446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017987.542452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86dce00 x1818247335446272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335446272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335446272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017987.542479:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335446400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017987.542491:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017987.542497:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017987/real 1734017987] req@ffff8800a86de300 x1818247335446400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017987.542507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335446400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017987.542514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017987.542516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017987.542519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335446400:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017987.947935:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017988.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017988.949967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017989.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017989.951933:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017990.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017990.953968:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017991.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017991.955968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017992.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017992.549828:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017992.549837:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017992.549842:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.549852:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.549860:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.549868:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017992.549904:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335446528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017992.549937:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017992.549941:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017992.549943:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.549947:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.549951:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.549954:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017992.549964:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017992.549975:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335446656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017992.549984:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017992.549987:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017992.549989:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.549993:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.549996:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.549999:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017992.550004:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017992.550012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335446784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017992.550020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017992.550023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017992.550024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.550028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.550030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.550033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017992.550037:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017992.550047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335446912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017992.550056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017992.550058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335446528:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017992.550059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017992.550060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.550064:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.550067:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.550070:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017992.550074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017992.550082:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335447040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017992.550090:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017992.550093:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017992.550095:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.550098:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.550100:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.550104:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017992.550107:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017992.550114:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335447168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017992.550122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017992.550125:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017992.550126:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017992.550129:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2257 00000100:00080000:2.0:1734017992.550132:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017992.550135:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017992.550139:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017992.550146:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335447296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017992.550151:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017992.550173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017992.550197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017992.550212:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017992.550227:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017992.550241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447296:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017992.957940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017993.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017993.960001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017994.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017994.961968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017995.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:0.0:1734017995.555992:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:0.0:1734017995.556047:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1734017995.556071:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017995.556114:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017995.556123:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017995.556131:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017995.556138:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017995.556145:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:0.0:1734017995.556152:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:1.0:1734017995.556242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86df100 x1818247335446912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.556283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335446912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.556302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.556308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.556316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017995.556329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86dc700 x1818247335447040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.556349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335447040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.556361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.556366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.556372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017995.556383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86de680 x1818247335447168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.556401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335447168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.556414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.556419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.556424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017995.556435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86dd500 x1818247335447296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.556454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335447296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.556467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.556471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.556476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017995.556488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86dd180 x1818247335446528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.556507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335446528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.556519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.556522:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.556528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335446528:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017995.556605:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86dc000 x1818247335446656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.556625:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335446656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.556656:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.556661:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.556667:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017995.556678:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017992/real 1734017995] req@ffff8800a86ddc00 x1818247335446784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017995.557172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335446784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017995.557182:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017995.557184:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017995.557187:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335446784:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017995.963904:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017996.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017996.965951:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017997.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734017997.557839:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017997.557849:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734017997.557854:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.557866:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.557876:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.557885:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734017997.557930:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335447424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017997.557968:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017997.557972:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017997.557974:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.557979:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.557982:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.557986:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017997.557997:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017997.558008:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335447552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017997.558018:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017997.558022:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017997.558024:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.558028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.558030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.558034:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017997.558038:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734017997.558047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335447680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017997.558057:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017997.558061:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734017997.558062:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.558067:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.558069:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.558072:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017997.558076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017997.558087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335447808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017997.558096:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734017997.558099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335447424:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734017997.558100:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734017997.558101:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.558106:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.558108:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.558112:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017997.558116:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734017997.558125:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335447936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017997.558134:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017997.558137:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734017997.558138:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.558142:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.558145:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.558149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017997.558153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017997.558161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335448064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734017997.558169:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734017997.558172:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1734017997.558173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734017997.558173:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734017997.558177:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2262 00000100:00080000:2.0:1734017997.558180:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734017997.558184:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734017997.558188:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734017997.558195:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335448192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734017997.558196:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86dca80 x1818247335447424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558217:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335447424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558236:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335447424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734017997.558244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335447552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017997.558262:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017997.558270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86dd880 x1818247335447552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558281:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335447552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335447552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017997.558301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335447680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017997.558315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017997.558321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86df480 x1818247335447680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335447680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558339:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335447680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017997.558370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335447808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017997.558384:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017997.558390:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86ddc00 x1818247335447808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335447808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335447808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017997.558428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734017997.558440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017997.558446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86dc000 x1818247335447936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335447936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335447936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734017997.558476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017997.558487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017997.558493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86dd180 x1818247335448064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335448064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558560:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558563:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734017997.558570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734017997.558586:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734017997.558593:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734017997/real 1734017997] req@ffff8800a86dd500 x1818247335448192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734017997.558603:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335448192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734017997.558611:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734017997.558613:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734017997.558616:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448192:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734017997.967924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017998.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017998.969955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734017999.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734017999.971865:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018000.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018000.973938:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018001.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018001.975953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018002.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018002.565760:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.565768:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018002.565772:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.565783:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00080000:2.0:1734018002.565791:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.565799:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018002.565836:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335448320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018002.565868:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.565872:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018002.565874:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.565878:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00080000:2.0:1734018002.565881:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.565885:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018002.565894:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018002.565905:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335448448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018002.565914:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.565917:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018002.565919:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.565923:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00080000:2.0:1734018002.565925:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.565929:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018002.565933:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018002.565941:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335448576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018002.565949:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.565952:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018002.565953:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.565957:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00080000:2.0:1734018002.565960:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.565963:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018002.565967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734018002.565975:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335448320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018002.565978:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335448704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018002.565987:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.565990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018002.565991:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.565995:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00080000:2.0:1734018002.565997:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.566000:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018002.566005:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018002.566012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335448832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018002.566020:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.566023:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018002.566025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.566028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00080000:2.0:1734018002.566032:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.566036:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018002.566040:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734018002.566044:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018002.566047:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335448960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018002.566056:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018002.566059:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018002.566060:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018002.566064:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2267 00000100:00100000:1.0:1734018002.566065:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86de680 x1818247335448320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018002.566067:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018002.566070:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018002.566074:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018002.566081:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335449088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018002.566084:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335448320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566094:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566098:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566102:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335448320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018002.566110:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335448448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018002.566127:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018002.566133:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86dc700 x1818247335448448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018002.566153:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335448448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566160:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566163:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566166:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335448448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018002.566172:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335448576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018002.566186:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018002.566192:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86df100 x1818247335448576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018002.566202:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335448576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566209:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566215:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335448576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018002.566241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335448704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018002.566253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018002.566260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86de300 x1818247335448704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018002.566271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335448704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335448704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018002.566290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018002.566303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018002.566309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86dce00 x1818247335448832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018002.566318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335448832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018002.566336:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018002.566348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018002.566354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86dea00 x1818247335448960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018002.566364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335448960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335448960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018002.566381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018002.566397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018002.566403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018002/real 1734018002] req@ffff8800a86dc380 x1818247335449088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018002.566461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335449088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018002.566470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018002.566472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018002.566476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449088:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018002.977941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018003.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018003.979962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018004.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018004.981971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018005.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018005.983973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018006.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018006.985932:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018007.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018007.573843:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.573851:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018007.573856:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.573866:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.573875:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018007.573882:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018007.573917:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335449216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018007.573950:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.573954:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018007.573957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.573961:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.573964:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018007.573968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018007.573977:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018007.573987:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335449344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018007.573997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.574000:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018007.574002:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.574006:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.574008:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018007.574012:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018007.574016:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018007.574023:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335449472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018007.574032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.574034:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018007.574036:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.574039:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.574042:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018007.574045:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018007.574049:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018007.574059:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335449600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018007.574061:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335449216:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018007.574069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.574072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018007.574074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.574077:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.574080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018007.574083:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018007.574087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018007.574094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335449728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018007.574102:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.574105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018007.574107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.574110:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.574112:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018007.574116:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018007.574119:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018007.574127:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335449856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734018007.574129:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018007.574135:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018007.574138:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018007.574140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018007.574143:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2272 00000100:00080000:2.0:1734018007.574147:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734018007.574150:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86df800 x1818247335449216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018007.574150:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018007.574154:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018007.574161:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335449984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018007.574169:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335449216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574180:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574183:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574188:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335449216:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018007.574195:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335449344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018007.574212:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018007.574218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86dfb80 x1818247335449344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018007.574229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335449344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335449344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018007.574250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335449472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018007.574268:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018007.574274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86ddf80 x1818247335449472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018007.574284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335449472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574297:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335449472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018007.574321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335449600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018007.574396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018007.574405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86dc380 x1818247335449600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018007.574415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335449600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335449600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018007.574436:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018007.574449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018007.574455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86dea00 x1818247335449728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018007.574466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335449728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018007.574484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018007.574495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018007.574501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86dce00 x1818247335449856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018007.574511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335449856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018007.574528:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018007.574540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018007.574546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018007/real 1734018007] req@ffff8800a86de300 x1818247335449984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018007.574557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335449984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018007.574565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018007.574567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018007.574570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335449984:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018007.987972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018008.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018008.989961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018009.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018009.991971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018010.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018010.994000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018011.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018011.995987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018012.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018012.581835:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.581845:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018012.581851:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018012.581864:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.581874:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018012.581884:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018012.581931:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335450112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018012.581971:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.581976:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018012.581978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018012.581984:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.581987:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018012.581991:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018012.582002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018012.582014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335450240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018012.582025:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.582028:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018012.582030:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018012.582033:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.582036:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018012.582039:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018012.582044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018012.582053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335450368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018012.582063:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.582067:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018012.582068:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018012.582072:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.582074:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018012.582078:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018012.582082:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018012.582092:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335450496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018012.582101:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.582105:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018012.582106:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734018012.582110:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335450112:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018012.582111:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.582114:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018012.582118:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018012.582123:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018012.582131:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335450624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018012.582139:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.582143:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018012.582144:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018012.582148:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.582150:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018012.582154:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018012.582158:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018012.582165:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335450752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018012.582173:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018012.582176:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018012.582178:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018012.582181:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2277 00000100:00080000:2.0:1734018012.582184:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1734018012.582188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018012.582188:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018012.582192:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018012.582200:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335450880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018012.582213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86df100 x1818247335450112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335450112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335450112:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018012.582297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335450240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018012.582320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018012.582328:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86dc700 x1818247335450240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582340:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335450240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335450240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018012.582362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335450368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018012.582376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018012.582383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86de680 x1818247335450368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335450368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335450368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018012.582434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335450496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018012.582446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018012.582453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86dd500 x1818247335450496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335450496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335450496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018012.582483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335450624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018012.582499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018012.582505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86dd180 x1818247335450624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335450624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335450624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018012.582534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335450752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018012.582545:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018012.582551:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86dc000 x1818247335450752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582561:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335450752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335450752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018012.582580:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335450880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018012.582592:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018012.582598:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018012/real 1734018012] req@ffff8800a86ddc00 x1818247335450880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018012.582615:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335450880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018012.582622:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018012.582624:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018012.582627:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335450880:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018012.997968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018013.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018013.999972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018014.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018015.001961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018015.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018016.003942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018016.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018017.005946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018017.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018017.589848:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.589856:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018017.589860:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018017.589869:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.589878:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.589886:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018017.589919:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335451008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018017.589951:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.589955:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018017.589957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018017.589962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.589965:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.589968:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018017.589978:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018017.589989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335451136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018017.589998:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.590002:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018017.590004:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018017.590008:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.590010:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.590014:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018017.590018:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018017.590026:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335451264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018017.590034:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.590037:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018017.590038:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018017.590041:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.590044:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.590047:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018017.590051:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734018017.590056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335451008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018017.590061:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335451392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018017.590069:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.590073:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018017.590074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018017.590077:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.590080:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.590083:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018017.590087:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018017.590094:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335451520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018017.590103:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.590106:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018017.590107:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018017.590111:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.590113:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.590117:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018017.590120:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734018017.590123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018017.590128:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335451648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018017.590136:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018017.590139:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018017.590140:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734018017.590144:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86df480 x1818247335451008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018017.590144:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2282 00000100:00080000:2.0:1734018017.590146:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018017.590149:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018017.590153:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018017.590190:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335451776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018017.590195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335451008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590207:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590210:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590216:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335451008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018017.590223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335451136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018017.590243:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018017.590251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86dd880 x1818247335451136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018017.590262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335451136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335451136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018017.590282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335451264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018017.590296:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018017.590302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86dca80 x1818247335451264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018017.590312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335451264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335451264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018017.590350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335451392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018017.590361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018017.590368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86ddc00 x1818247335451392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018017.590378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335451392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335451392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018017.590396:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335451520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018017.590412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018017.590418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86dc000 x1818247335451520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018017.590428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335451520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590438:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335451520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018017.590446:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335451648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018017.590458:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018017.590463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86dd180 x1818247335451648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018017.590473:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335451648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335451648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018017.590491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335451776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018017.590502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018017.590508:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018017/real 1734018017] req@ffff8800a86dd500 x1818247335451776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018017.590524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335451776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018017.590532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018017.590534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018017.590536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335451776:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018018.007947:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018018.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018019.009924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018019.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018020.011975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018020.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018021.013951:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018021.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018022.015959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018022.213763:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018022.597796:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597804:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018022.597807:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597813:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00080000:2.0:1734018022.597820:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597825:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018022.597850:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335451904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018022.597871:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597873:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018022.597874:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597877:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00080000:2.0:1734018022.597879:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597881:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018022.597887:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018022.597893:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335452032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018022.597898:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597900:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018022.597900:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597903:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00080000:2.0:1734018022.597904:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597906:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018022.597908:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018022.597912:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335452160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018022.597918:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597919:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1734018022.597920:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335451904:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018022.597920:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597922:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00080000:2.0:1734018022.597924:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597925:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018022.597928:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018022.597933:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335452288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018022.597938:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597939:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018022.597940:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597942:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00080000:2.0:1734018022.597943:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597945:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018022.597947:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018022.597951:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335452416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018022.597955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597957:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018022.597958:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597960:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00080000:2.0:1734018022.597962:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597964:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:1.0:1734018022.597965:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734018022.597966:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018022.597969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335452544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018022.597973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018022.597975:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018022.597976:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018022.597977:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2287 00000100:00100000:1.0:1734018022.597978:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86de680 x1818247335451904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018022.597979:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018022.597980:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018022.597982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018022.597986:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335452672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018022.597989:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335451904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.597995:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.597996:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.597999:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335451904:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018022.598003:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018022.598013:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018022.598017:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86dc700 x1818247335452032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018022.598023:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335452032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.598027:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.598029:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.598030:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018022.598044:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018022.598051:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018022.598055:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86df100 x1818247335452160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018022.598061:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335452160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.598065:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.598066:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.598068:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018022.598094:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018022.598105:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018022.598109:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86de300 x1818247335452288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018022.598114:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335452288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.598119:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.598120:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.598121:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018022.598125:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335452416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018022.598133:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018022.598137:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86dce00 x1818247335452416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018022.598142:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335452416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.598146:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.598147:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.598149:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335452416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018022.598152:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335452544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018022.598158:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018022.598161:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86dea00 x1818247335452544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018022.598166:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335452544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.598170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.598171:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.598172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335452544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018022.598175:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335452672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018022.598181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018022.598184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018022/real 1734018022] req@ffff8800a86dc380 x1818247335452672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018022.598193:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335452672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018022.598197:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018022.598198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018022.598199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335452672:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018023.017885:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018023.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018024.019887:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018024.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018025.021935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018025.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018026.023893:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018026.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018027.025869:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018027.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018027.605795:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018027.605803:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018027.605807:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018027.605816:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.605823:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.605830:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018027.605855:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335452800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018027.605886:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018027.605889:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018027.605891:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018027.605894:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.605897:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.605899:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018027.605906:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018027.605915:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335452928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018027.605923:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018027.605925:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018027.605927:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018027.605929:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.605931:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.605934:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734018027.605935:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335452800:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734018027.605938:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018027.605944:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335453056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018027.605951:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018027.605954:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018027.605955:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018027.605958:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.605960:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.605963:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018027.605966:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018027.605974:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335453184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018027.605981:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018027.605983:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018027.605985:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1734018027.605997:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018027.605997:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.606000:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.606003:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018027.606006:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1734018027.606012:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86ddf80 x1818247335452800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018027.606014:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335453312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018027.606021:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018027.606024:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018027.606025:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1734018027.606028:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335452800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018027.606028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.606030:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.606033:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018027.606036:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1734018027.606037:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606039:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018027.606042:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335453440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018027.606043:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335452800:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018027.606049:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1734018027.606050:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452928:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734018027.606051:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018027.606053:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018027.606056:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2292 00000100:00080000:2.0:1734018027.606058:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018027.606060:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018027.606063:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734018027.606065:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018027.606069:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335453568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018027.606072:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86dfb80 x1818247335452928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018027.606082:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335452928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018027.606088:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606091:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018027.606094:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335452928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018027.606112:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018027.606123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018027.606129:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86df800 x1818247335453056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018027.606138:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335453056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018027.606144:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606146:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018027.606149:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018027.606154:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018027.606164:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018027.606169:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86dc380 x1818247335453184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018027.606178:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335453184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018027.606185:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606187:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018027.606189:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018027.606194:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335453312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018027.606206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018027.606212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86dea00 x1818247335453312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018027.606221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335453312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018027.606227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018027.606232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335453312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018027.606236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335453440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018027.606245:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018027.606250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86dce00 x1818247335453440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018027.606259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335453440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018027.606265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018027.606270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335453440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018027.606274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335453568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018027.606284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018027.606289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018027/real 1734018027] req@ffff8800a86de300 x1818247335453568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018027.606303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335453568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018027.606310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018027.606311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018027.606314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335453568:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018028.027872:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018028.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018029.029841:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018029.213900:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018030.031840:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018030.213729:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018031.033970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018031.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018032.035941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018032.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018032.613815:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.613825:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018032.613829:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.613837:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.613845:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.613852:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018032.613886:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335453696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018032.613925:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.613928:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018032.613928:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.613931:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.613933:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.613936:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018032.613943:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018032.613950:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335453824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018032.613955:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.613956:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018032.613957:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.613960:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.613961:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.613962:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018032.613965:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018032.613969:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335453952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018032.613973:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.613974:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018032.613975:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.613977:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.613978:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.613980:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018032.613982:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018032.613989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335454080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018032.613995:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335453696:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018032.613997:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.613999:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018032.614000:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.614002:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.614003:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.614005:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018032.614007:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018032.614011:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335454208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018032.614014:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.614015:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018032.614016:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.614018:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.614019:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.614021:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018032.614023:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018032.614028:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335454336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018032.614032:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018032.614033:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018032.614034:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018032.614035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2297 00000100:00080000:2.0:1734018032.614037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018032.614039:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018032.614041:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018032.614046:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335454464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734018032.614050:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614066:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86df100 x1818247335453696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614078:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335453696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614084:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614085:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614088:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335453696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018032.614092:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018032.614102:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614105:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86dc700 x1818247335453824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614110:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335453824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614114:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614116:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614117:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018032.614120:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018032.614128:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614130:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86de680 x1818247335453952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614136:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335453952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614147:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614149:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614151:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335453952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018032.614174:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018032.614187:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614193:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86dd500 x1818247335454080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614201:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335454080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614209:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018032.614216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335454208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018032.614231:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86dd180 x1818247335454208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335454208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335454208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018032.614258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335454336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018032.614269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86dc000 x1818247335454336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335454336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335454336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018032.614296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335454464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018032.614306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018032.614310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018032/real 1734018032] req@ffff8800a86ddc00 x1818247335454464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018032.614322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335454464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018032.614328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018032.614329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018032.614331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335454464:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018033.037951:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018033.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018034.039892:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018034.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018035.041905:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018035.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018036.043876:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018036.213883:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018037.045862:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018037.213787:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018037.621852:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.621863:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018037.621868:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.621879:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.621889:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018037.621897:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018037.621934:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335454592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018037.621976:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.621979:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018037.621981:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.621985:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.621988:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018037.621992:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018037.622002:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018037.622013:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335454720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018037.622027:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.622029:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018037.622031:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.622035:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.622037:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018037.622040:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018037.622044:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018037.622053:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335454848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018037.622060:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.622063:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018037.622064:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.622067:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.622070:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018037.622073:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018037.622076:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018037.622087:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335454976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018037.622094:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.622097:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018037.622098:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.622103:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.622105:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018037.622108:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018037.622112:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018037.622119:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335455104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018037.622127:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.622129:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018037.622131:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.622134:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.622136:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018037.622139:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018037.622142:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018037.622149:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335455232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018037.622155:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018037.622157:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018037.622158:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018037.622161:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2302 00000100:00080000:2.0:1734018037.622163:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734018037.622164:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335454592:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018037.622165:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018037.622169:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018037.622176:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335455360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734018037.622230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86dca80 x1818247335454592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335454592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335454592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018037.622286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018037.622301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86dd880 x1818247335454720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335454720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622321:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622324:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018037.622331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018037.622342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86df480 x1818247335454848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335454848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018037.622371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018037.622384:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622388:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86ddc00 x1818247335454976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335454976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335454976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018037.622404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335455104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018037.622414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86dc000 x1818247335455104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622424:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335455104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622429:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335455104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018037.622453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335455232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018037.622466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86dd180 x1818247335455232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335455232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335455232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018037.622497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335455360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018037.622509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018037.622514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018037/real 1734018037] req@ffff8800a86dd500 x1818247335455360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018037.622528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335455360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018037.622535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018037.622537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018037.622539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335455360:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018038.047883:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018038.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018039.049947:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018039.213754:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018040.051872:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018040.213897:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018041.053865:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018041.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018042.055916:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018042.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018042.629816:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629823:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018042.629826:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629833:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.629840:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018042.629845:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018042.629870:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335455488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018042.629891:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629893:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018042.629894:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629897:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.629899:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018042.629901:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018042.629907:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018042.629912:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335455616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018042.629917:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629919:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018042.629920:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629922:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.629924:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018042.629925:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018042.629928:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018042.629933:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335455744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018042.629937:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629939:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018042.629939:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629941:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.629943:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018042.629944:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018042.629946:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018042.629952:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335455872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018042.629957:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629959:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018042.629959:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629962:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.629963:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1734018042.629964:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335455488:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018042.629965:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018042.629967:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018042.629971:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335456000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018042.629975:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629977:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018042.629978:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629980:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.629982:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018042.629984:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018042.629986:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018042.629989:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335456128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018042.629993:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018042.629995:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018042.629995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018042.629997:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2307 00000100:00080000:2.0:1734018042.630004:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018042.630006:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018042.630008:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1734018042.630009:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018042.630012:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335456256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018042.630022:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86de680 x1818247335455488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630032:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335455488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630038:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630040:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630042:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335455488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018042.630046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335455616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018042.630055:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018042.630058:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86dc700 x1818247335455616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630064:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335455616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630068:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630069:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630071:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335455616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018042.630074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335455744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018042.630082:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018042.630085:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86df100 x1818247335455744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630090:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335455744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630094:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630095:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630097:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335455744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018042.630110:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335455872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018042.630116:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018042.630120:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86de300 x1818247335455872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630125:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335455872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630129:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630130:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630132:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335455872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018042.630134:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018042.630142:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018042.630145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86dce00 x1818247335456000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630151:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335456000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630154:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630156:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630157:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018042.630160:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018042.630166:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018042.630169:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86dea00 x1818247335456128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630174:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335456128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630178:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630179:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630180:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018042.630183:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018042.630189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018042.630192:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018042/real 1734018042] req@ffff8800a86dc380 x1818247335456256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018042.630200:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335456256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018042.630203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018042.630204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018042.630206:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456256:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018043.057841:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018043.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018044.059915:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018044.213921:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018045.062053:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018045.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018046.064019:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018046.213898:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018047.066064:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018047.213933:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018047.637869:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.637882:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018047.637887:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.637900:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.637910:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.637919:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018047.637956:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335456384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018047.637990:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.637993:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018047.637994:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.637997:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.637999:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.638002:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018047.638010:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018047.638018:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335456512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018047.638023:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.638025:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018047.638026:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.638028:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.638029:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.638031:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018047.638034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018047.638038:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335456640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018047.638043:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.638044:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018047.638045:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.638047:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.638048:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.638051:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018047.638054:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018047.638063:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335456768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018047.638067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.638069:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018047.638070:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.638072:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.638073:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.638075:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018047.638078:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018047.638083:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335456896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018047.638088:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.638090:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018047.638091:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.638093:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.638095:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.638097:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018047.638099:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1734018047.638107:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335456384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018047.638111:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335457024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018047.638115:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018047.638118:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018047.638121:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018047.638124:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2312 00000100:00080000:2.0:1734018047.638126:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018047.638128:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018047.638131:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018047.638136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335457152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734018047.638168:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638186:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86df800 x1818247335456384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638200:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335456384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638211:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638214:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335456384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:1.0:1734018047.638219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335456512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018047.638230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638234:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86dfb80 x1818247335456512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638241:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335456512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335456512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018047.638255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335456640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018047.638264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86ddf80 x1818247335456640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335456640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638279:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335456640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018047.638287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335456768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018047.638295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86dc380 x1818247335456768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638305:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335456768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335456768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018047.638345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018047.638356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638360:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86dea00 x1818247335456896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335456896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335456896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018047.638379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018047.638387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86dce00 x1818247335457024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335457024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018047.638409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018047.638427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018047.638431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018047/real 1734018047] req@ffff8800a86de300 x1818247335457152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018047.638441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335457152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018047.638446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018047.638447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018047.638449:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457152:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018048.067994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018048.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018049.069931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018049.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018050.071900:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018050.213772:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018051.073914:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018051.213792:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018052.075992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018052.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018052.645852:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.645866:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018052.645872:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018052.645885:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00080000:2.0:1734018052.645897:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018052.645907:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018052.645949:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335457280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018052.645995:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.646001:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018052.646003:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018052.646010:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00080000:2.0:1734018052.646015:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018052.646021:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018052.646034:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018052.646051:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335457408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018052.646067:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.646072:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018052.646074:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018052.646080:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00080000:2.0:1734018052.646084:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018052.646089:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018052.646095:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018052.646107:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335457536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018052.646116:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335457280:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018052.646122:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.646126:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018052.646129:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018052.646134:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00080000:2.0:1734018052.646138:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018052.646143:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018052.646149:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018052.646164:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335457664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018052.646178:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.646182:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:1.0:1734018052.646184:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018052.646185:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018052.646191:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00080000:2.0:1734018052.646196:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018052.646201:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86df100 x1818247335457280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734018052.646207:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018052.646218:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335457792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018052.646226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335457280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018052.646231:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.646235:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1734018052.646238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018052.646238:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1734018052.646241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734018052.646243:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00100000:1.0:1734018052.646246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335457280:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018052.646247:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018052.646253:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335457408:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734018052.646259:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018052.646270:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335457920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:1.0:1734018052.646271:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646278:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86dc700 x1818247335457408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018052.646283:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018052.646287:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1734018052.646290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335457408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018052.646290:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018052.646295:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2317 00000100:00080000:1.0:1734018052.646298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018052.646300:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1734018052.646301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018052.646305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335457408:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734018052.646305:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018052.646310:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018052.646321:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335458048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:1.0:1734018052.646329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335457536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018052.646343:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86de680 x1818247335457536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018052.646360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335457536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018052.646368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018052.646370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018052.646373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335457536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018052.646379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335457664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018052.646390:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646396:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86dd500 x1818247335457664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018052.646405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335457664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018052.646413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018052.646415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018052.646419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335457664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018052.646424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:1.0:1734018052.646439:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86dd180 x1818247335457792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018052.646455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335457792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018052.646463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018052.646465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018052.646468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:1.0:1734018052.646473:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018052.646484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86dc000 x1818247335457920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018052.646500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335457920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018052.646507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018052.646509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018052.646513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335457920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:1.0:1734018052.646527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:1.0:1734018052.646539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:1.0:1734018052.646545:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018052/real 1734018052] req@ffff8800a86ddc00 x1818247335458048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:1.0:1734018052.646621:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335458048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:1.0:1734018052.646630:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1734018052.646632:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1734018052.646635:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458048:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018053.077963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018053.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018054.079888:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018054.213782:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018055.081842:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018055.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018056.083907:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018056.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018057.085941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018057.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018057.653977:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018057.653990:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018057.653995:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018057.654006:0:9726:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654016:0:9726:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654024:0:9726:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018057.654060:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335458176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018057.654100:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018057.654103:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018057.654105:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018057.654109:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654112:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654116:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018057.654125:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018057.654127:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335458176:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018057.654136:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335458304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018057.654145:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018057.654148:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018057.654151:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018057.654154:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654157:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654160:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018057.654164:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018057.654174:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335458432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018057.654181:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018057.654184:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018057.654185:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018057.654189:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654191:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654195:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018057.654198:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018057.654205:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335458560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018057.654213:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018057.654216:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018057.654217:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1734018057.654219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335458304:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734018057.654220:0:9726:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654223:0:9726:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654226:0:9726:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018057.654229:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018057.654236:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335458688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018057.654245:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734018057.654246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335458432:192.168.201.104@tcp:8:kworker.0 00000100:00080000:0.0:1734018057.654248:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018057.654249:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018057.654253:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654255:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654258:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018057.654261:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018057.654268:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335458816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018057.654268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335458560:192.168.201.104@tcp:8:kworker.0 00000100:00080000:0.0:1734018057.654275:0:9726:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018057.654278:0:9726:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018057.654279:0:9726:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018057.654282:0:9726:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2322 00000100:00080000:0.0:1734018057.654285:0:9726:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018057.654287:0:9726:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018057.654291:0:9726:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018057.654292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018057.654298:0:9726:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335458944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018057.654315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018057.654337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458944:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018058.087992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018058.213897:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018059.089903:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018059.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018060.091852:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018060.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734018060.659936:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018060.659979:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018060.659995:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018060.660029:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018060.660037:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018060.660044:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734018060.660047:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59cb100 x1818247335458304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000800:00000100:1.0:1734018060.660051:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018060.660058:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018060.660063:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:0.0:1734018060.660074:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335458304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660083:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660086:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660090:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335458304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018060.660096:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59cb800 x1818247335458432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018060.660106:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335458432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660113:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660115:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660118:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335458432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018060.660123:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59cad80 x1818247335458560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018060.660131:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335458560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660137:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660139:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660142:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335458560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018060.660148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59c9180 x1818247335458176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018060.660163:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335458176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660170:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660172:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660175:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335458176:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018060.660199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59ca300 x1818247335458944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018060.660209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335458944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660219:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018060.660224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59c9500 x1818247335458688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018060.660233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335458688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018060.660249:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018057/real 1734018060] req@ffff8800b59c8000 x1818247335458816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018060.660262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335458816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018060.660268:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018060.660270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018060.660273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335458816:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018061.093885:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018061.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018062.096034:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018062.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018062.661801:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018062.661809:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018062.661812:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661820:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661826:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661831:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018062.661857:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335459072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018062.661881:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018062.661883:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018062.661884:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661887:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661888:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661890:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018062.661896:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018062.661902:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335459200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018062.661908:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018062.661910:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018062.661910:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661912:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661913:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661915:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018062.661918:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018062.661922:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335459328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018062.661927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734018062.661927:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335459072:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018062.661928:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018062.661929:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661931:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661932:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661934:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018062.661936:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018062.661940:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335459456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018062.661944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018062.661945:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018062.661946:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661948:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661950:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661952:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018062.661954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018062.661958:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335459584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018062.661962:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018062.661964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018062.661964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661966:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661967:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661969:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018062.661971:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734018062.661973:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018062.661975:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335459712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018062.661979:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018062.661980:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018062.661981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018062.661983:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2327 00000100:00080000:0.0:1734018062.661984:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018062.661986:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734018062.661986:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59c8000 x1818247335459072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018062.661988:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018062.661991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335459840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018062.661997:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335459072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662003:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662005:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662007:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335459072:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018062.662011:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335459200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018062.662020:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018062.662024:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59c9500 x1818247335459200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018062.662030:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335459200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662034:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662036:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662037:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335459200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018062.662053:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335459328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018062.662060:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018062.662063:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59ca300 x1818247335459328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018062.662068:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335459328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662073:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662075:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662077:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335459328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018062.662082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335459456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018062.662094:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018062.662099:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59c9180 x1818247335459456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018062.662108:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335459456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662120:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662121:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662124:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335459456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018062.662129:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335459584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018062.662144:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018062.662150:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59cad80 x1818247335459584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018062.662159:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335459584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662165:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662170:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335459584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018062.662174:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335459712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018062.662186:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018062.662191:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59cb800 x1818247335459712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018062.662199:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335459712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662206:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662208:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662210:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335459712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018062.662215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335459840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018062.662227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018062.662239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018062/real 1734018062] req@ffff8800b59cb100 x1818247335459840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018062.662249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335459840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018062.662255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018062.662257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018062.662259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335459840:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018063.097919:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018063.213781:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018064.099853:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018064.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018065.102021:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018065.213926:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018066.103902:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018066.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018067.105974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018067.213788:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018067.669802:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669809:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018067.669812:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669819:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669825:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669831:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018067.669857:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335459968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018067.669879:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669881:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018067.669882:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669885:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669887:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669889:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018067.669895:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018067.669901:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335460096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018067.669906:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669907:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018067.669908:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669910:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669913:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669915:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018067.669917:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018067.669922:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335460224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018067.669926:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669927:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018067.669928:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669930:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669931:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669933:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018067.669935:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018067.669939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335460352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018067.669943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669945:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018067.669946:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669948:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669949:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669951:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018067.669953:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018067.669953:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335459968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018067.669957:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335460480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018067.669964:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669966:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018067.669967:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669968:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669969:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669971:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018067.669973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018067.669977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335460608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018067.669981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018067.669983:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018067.669983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018067.669985:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2332 00000100:00080000:0.0:1734018067.669986:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018067.669988:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018067.669990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018067.669994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335460736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:2.0:1734018067.669999:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670012:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59ca680 x1818247335459968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670024:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335459968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670030:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670032:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670034:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335459968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018067.670038:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018067.670049:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670053:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59c8e00 x1818247335460096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670059:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335460096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670063:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670064:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670066:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018067.670068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018067.670075:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670078:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59c8a80 x1818247335460224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670083:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335460224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670087:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670088:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670089:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018067.670092:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018067.670099:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670102:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59c8700 x1818247335460352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670110:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335460352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670114:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670115:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670117:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018067.670134:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335460480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018067.670140:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670143:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59c9f80 x1818247335460480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670148:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335460480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670152:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670153:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670155:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335460480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018067.670158:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335460608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018067.670163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59cb480 x1818247335460608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670171:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335460608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670175:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670176:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670178:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335460608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018067.670180:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335460736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018067.670186:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018067.670189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018067/real 1734018067] req@ffff8800b59cbb80 x1818247335460736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018067.670194:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335460736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018067.670197:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018067.670198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018067.670200:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335460736:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018068.107999:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018068.213799:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018069.109901:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018069.213769:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018070.111979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018070.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018071.113842:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018071.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018072.115940:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018072.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018072.677842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.677853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018072.677857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.677868:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.677876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.677885:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018072.677921:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335460864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018072.677956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.677960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018072.677962:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.677967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.677969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.677973:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018072.677982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018072.677993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335460992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018072.678002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.678006:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018072.678007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.678011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.678014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.678017:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018072.678022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018072.678031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335461120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018072.678040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.678043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018072.678045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.678049:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.678052:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.678055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018072.678059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018072.678067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335461248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018072.678068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335460864:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018072.678076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.678079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018072.678080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.678094:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.678097:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.678100:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018072.678104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018072.678112:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335461376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018072.678120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.678122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018072.678124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.678127:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.678130:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.678133:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018072.678137:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734018072.678137:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018072.678145:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335461504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018072.678154:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018072.678157:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1734018072.678158:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59caa00 x1818247335460864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018072.678159:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018072.678162:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2337 00000100:00080000:0.0:1734018072.678165:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018072.678168:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018072.678172:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018072.678179:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335460864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018072.678180:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335461632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018072.678189:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678194:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335460864:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018072.678206:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018072.678251:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018072.678260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59c9c00 x1818247335460992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018072.678271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335460992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018072.678280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678283:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335460992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018072.678293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335461120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018072.678308:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018072.678314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59cbb80 x1818247335461120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018072.678324:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335461120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018072.678332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678334:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678337:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335461120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018072.678362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335461248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018072.678378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018072.678384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59cb480 x1818247335461248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018072.678395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335461248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018072.678402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335461248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018072.678413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335461376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018072.678425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018072.678430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59c9f80 x1818247335461376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018072.678440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335461376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018072.678447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335461376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018072.678458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335461504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018072.678470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018072.678475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59c8700 x1818247335461504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018072.678492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335461504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018072.678500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335461504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018072.678510:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335461632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018072.678521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018072.678527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018072/real 1734018072] req@ffff8800b59c8a80 x1818247335461632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018072.678536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335461632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018072.678543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018072.678545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018072.678548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335461632:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018073.117958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018073.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018074.119940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018074.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018075.121948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018075.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018076.123948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018076.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018077.125945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018077.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018077.685835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018077.685845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018077.685850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.685860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.685870:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.685878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018077.685913:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335461760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018077.685947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018077.685951:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018077.685953:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.685958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.685961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.685965:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018077.685976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018077.685987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335461888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018077.685998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018077.686001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018077.686003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.686007:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.686009:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.686013:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018077.686017:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018077.686024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335462016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018077.686032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018077.686035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018077.686037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.686040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.686042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.686046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686049:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335461760:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018077.686050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018077.686057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335462144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018077.686066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018077.686069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018077.686072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.686075:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.686077:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.686080:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018077.686084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018077.686091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335462272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018077.686100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018077.686102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018077.686104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.686107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.686110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.686113:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:2.0:1734018077.686114:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734018077.686117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018077.686124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335462400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018077.686132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734018077.686166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59c8e00 x1818247335461760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018077.686172:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018077.686174:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018077.686179:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2342 00000100:00080000:0.0:1734018077.686182:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018077.686185:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335461760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018077.686189:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1734018077.686198:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1734018077.686201:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335462528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018077.686202:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335461760:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018077.686215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335461888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018077.686235:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59ca680 x1818247335461888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018077.686254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335461888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018077.686262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018077.686265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335461888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018077.686273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018077.686287:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59cb100 x1818247335462016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018077.686303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335462016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018077.686310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018077.686313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018077.686343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018077.686355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59cb800 x1818247335462144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018077.686372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335462144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018077.686379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018077.686381:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018077.686390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335462272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018077.686401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686407:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59cad80 x1818247335462272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018077.686417:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335462272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018077.686424:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018077.686426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335462272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018077.686435:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335462400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018077.686449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59c9180 x1818247335462400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018077.686465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335462400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018077.686473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018077.686475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335462400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018077.686483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335462528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018077.686495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018077.686500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018077/real 1734018077] req@ffff8800b59ca300 x1818247335462528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018077.686510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335462528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018077.686517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018077.686519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018077.686522:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335462528:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018078.127949:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018078.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018079.129941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018079.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018080.131925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018080.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018081.133974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018081.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018082.135950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018082.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018082.693929:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018082.693938:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018082.693942:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018082.693952:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.693961:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.693969:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018082.694005:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335462656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018082.694041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018082.694046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018082.694082:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018082.694089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.694092:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.694096:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018082.694106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018082.694118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335462784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018082.694129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018082.694132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018082.694134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018082.694137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.694140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.694143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694147:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335462656:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734018082.694148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018082.694155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335462912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018082.694165:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018082.694168:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018082.694170:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018082.694173:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.694176:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.694179:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018082.694183:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018082.694192:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335463040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018082.694200:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018082.694203:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018082.694204:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018082.694208:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.694210:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.694214:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:0.0:1734018082.694216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734018082.694218:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018082.694226:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335463168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018082.694234:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734018082.694237:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86df480 x1818247335462656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018082.694237:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018082.694238:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018082.694242:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.694245:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.694249:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018082.694253:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018082.694256:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335462656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018082.694260:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335463296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018082.694266:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018082.694268:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018082.694270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734018082.694271:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018082.694272:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734018082.694275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335462656:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018082.694276:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2347 00000100:00080000:2.0:1734018082.694278:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018082.694281:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462784:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734018082.694285:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018082.694292:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335463424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018082.694299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86dd880 x1818247335462784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018082.694318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335462784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018082.694325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018082.694328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018082.694331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018082.694356:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018082.694369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86dca80 x1818247335462912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018082.694386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335462912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018082.694394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018082.694397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018082.694400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335462912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018082.694405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018082.694417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86ddc00 x1818247335463040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018082.694432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335463040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018082.694439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018082.694441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018082.694444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018082.694450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335463168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018082.694462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86dc000 x1818247335463168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018082.694479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335463168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018082.694486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018082.694490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018082.694493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335463168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018082.694498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335463296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018082.694510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86dd180 x1818247335463296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018082.694526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335463296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018082.694533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018082.694535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018082.694538:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335463296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018082.694544:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335463424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018082.694556:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018082.694561:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018082/real 1734018082] req@ffff8800a86dd500 x1818247335463424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018082.694571:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335463424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018082.694578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018082.694581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018082.694584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335463424:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018083.137981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018083.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018084.139976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018084.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018085.141952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018085.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018086.143979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018086.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018087.145946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018087.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018087.701836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018087.701845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018087.701850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.701860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.701869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018087.701877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018087.701914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335463552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018087.701947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018087.701951:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018087.701953:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.701958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.701961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018087.702020:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018087.702030:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018087.702043:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335463680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018087.702055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734018087.702058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335463552:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018087.702058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018087.702060:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.702064:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.702068:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018087.702071:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018087.702076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018087.702083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335463808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018087.702092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018087.702095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018087.702096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.702101:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.702103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018087.702106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018087.702110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018087.702119:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335463936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018087.702127:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018087.702130:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1734018087.702132:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018087.702132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.702135:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.702138:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018087.702141:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018087.702145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018087.702153:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86de680 x1818247335463552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018087.702153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335464064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018087.702161:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018087.702164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018087.702165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.702169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.702171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734018087.702172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335463552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018087.702175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018087.702179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1734018087.702182:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702186:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018087.702186:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335464192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018087.702191:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335463552:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018087.702194:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018087.702197:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018087.702199:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018087.702203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2352 00000100:00080000:2.0:1734018087.702205:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018087.702208:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018087.702212:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018087.702215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018087.702220:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335464320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018087.702233:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018087.702240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86dc700 x1818247335463680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018087.702252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335463680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018087.702259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018087.702265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018087.702271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018087.702285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018087.702291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86df100 x1818247335463808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018087.702301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335463808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018087.702310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018087.702316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018087.702321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018087.702332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018087.702338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86de300 x1818247335463936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018087.702348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335463936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018087.702356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018087.702361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335463936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018087.702367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018087.702379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018087.702385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86dce00 x1818247335464064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018087.702395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335464064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018087.702403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018087.702408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018087.702415:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018087.702427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018087.702433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86dea00 x1818247335464192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018087.702443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335464192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018087.702451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018087.702455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018087.702474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018087.702486:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018087.702492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018087/real 1734018087] req@ffff8800a86dc380 x1818247335464320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018087.702502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335464320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018087.702509:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018087.702511:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018087.702514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464320:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018088.148031:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018088.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018089.149970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018089.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018090.152026:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018090.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018091.154007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018091.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018092.155972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018092.213779:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018092.709844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.709854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018092.709859:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.709868:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.709907:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.709916:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018092.709954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335464448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018092.709985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.709990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018092.709992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.709997:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.710000:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.710004:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018092.710014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018092.710024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335464576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018092.710034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.710037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018092.710039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.710042:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.710045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.710048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018092.710052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018092.710060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335464704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018092.710069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.710072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018092.710073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.710077:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.710080:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.710083:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018092.710087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018092.710097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335464832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018092.710100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335464448:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018092.710107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.710110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018092.710112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.710115:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.710117:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.710121:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018092.710124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018092.710132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335464960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018092.710140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.710142:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018092.710144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.710147:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.710150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.710153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018092.710157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018092.710164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335465088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018092.710169:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018092.710171:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018092.710174:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018092.710176:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018092.710179:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2357 00000100:00080000:2.0:1734018092.710184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018092.710187:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86ddf80 x1818247335464448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734018092.710191:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018092.710199:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335465216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018092.710209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335464448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335464448:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018092.710235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335464576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018092.710252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86dfb80 x1818247335464576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018092.710270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335464576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335464576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018092.710289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335464704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018092.710302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86df800 x1818247335464704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018092.710318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335464704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335464704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018092.710361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335464832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018092.710373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86dc380 x1818247335464832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018092.710390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335464832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335464832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018092.710408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018092.710419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86dea00 x1818247335464960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018092.710435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335464960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335464960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018092.710453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018092.710468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86dce00 x1818247335465088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018092.710484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335465088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018092.710501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018092.710512:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018092.710518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018092/real 1734018092] req@ffff8800a86de300 x1818247335465216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018092.710528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335465216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018092.710535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018092.710537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018092.710540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465216:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018093.157967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018093.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018094.159947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018094.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018095.162001:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018095.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018096.163987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018096.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018097.165990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018097.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018097.717854:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.717868:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018097.717874:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.717888:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.717900:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018097.717910:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018097.717958:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335465344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018097.717993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.717998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018097.718000:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.718005:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.718008:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018097.718012:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018097.718024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018097.718036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335465472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018097.718047:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.718051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018097.718053:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.718057:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.718060:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018097.718063:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018097.718068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018097.718077:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335465600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018097.718087:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.718090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018097.718091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.718095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.718098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018097.718101:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335465344:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734018097.718105:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018097.718116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335465728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018097.718124:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.718128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018097.718129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.718134:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.718136:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018097.718140:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018097.718144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018097.718152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335465856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018097.718160:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.718163:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018097.718165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.718169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.718171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018097.718175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018097.718179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018097.718187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335465984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018097.718190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018097.718195:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018097.718198:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018097.718199:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018097.718203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2362 00000100:00080000:2.0:1734018097.718214:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734018097.718215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86df100 x1818247335465344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018097.718218:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018097.718223:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018097.718231:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335466112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018097.718245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335465344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335465344:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018097.718272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335465472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018097.718290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86dc700 x1818247335465472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018097.718308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335465472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335465472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018097.718352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335465600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018097.718366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86de680 x1818247335465600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018097.718382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335465600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718397:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335465600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018097.718404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335465728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018097.718416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86dd500 x1818247335465728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018097.718432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335465728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335465728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018097.718449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018097.718462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86dd180 x1818247335465856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018097.718477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335465856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018097.718497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018097.718513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86dc000 x1818247335465984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018097.718529:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335465984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335465984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018097.718547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335466112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018097.718559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018097.718565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018097/real 1734018097] req@ffff8800a86ddc00 x1818247335466112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018097.718574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335466112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018097.718583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018097.718585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018097.718588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335466112:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018098.168040:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018098.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018099.169965:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018099.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018100.171996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018100.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018101.173978:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018101.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018102.176019:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018102.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018102.725850:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018102.725860:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018102.725866:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.725878:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.725889:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.725898:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018102.725946:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335466240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018102.725982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018102.725987:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018102.725989:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.725994:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.725997:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.726001:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018102.726013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018102.726024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335466368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018102.726034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018102.726037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018102.726040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.726044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.726047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.726051:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018102.726056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018102.726065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335466496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018102.726073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018102.726076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018102.726078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.726081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.726084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.726087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018102.726092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018102.726096:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335466240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018102.726102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335466624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018102.726111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018102.726115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018102.726116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.726120:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.726123:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.726126:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018102.726130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018102.726139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335466752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018102.726148:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018102.726152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018102.726153:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.726157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.726159:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.726163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018102.726167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018102.726174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335466880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018102.726182:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1734018102.726183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018102.726185:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018102.726187:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018102.726190:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2367 00000100:00080000:2.0:1734018102.726193:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018102.726196:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018102.726200:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018102.726207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86dca80 x1818247335466240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018102.726209:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335467008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018102.726237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335466240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335466240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018102.726265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335466368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018102.726282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018102.726288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86dd880 x1818247335466368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018102.726300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335466368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335466368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018102.726319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335466496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018102.726333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018102.726339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86df480 x1818247335466496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018102.726349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335466496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335466496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018102.726396:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335466624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018102.726409:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018102.726415:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86ddc00 x1818247335466624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018102.726426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335466624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335466624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018102.726444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335466752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018102.726456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018102.726461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86dc000 x1818247335466752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018102.726471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335466752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335466752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018102.726490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335466880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018102.726505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018102.726512:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86dd180 x1818247335466880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018102.726522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335466880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335466880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018102.726540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018102.726552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018102.726557:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018102/real 1734018102] req@ffff8800a86dd500 x1818247335467008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018102.726567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335467008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018102.726574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018102.726576:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018102.726579:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467008:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018103.177990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018103.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018104.179983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018104.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018105.181996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018105.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018106.183960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018106.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018107.185961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018107.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018107.733812:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.733822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018107.733827:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.733837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00080000:2.0:1734018107.733846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.733855:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018107.733892:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335467136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018107.733925:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.733929:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018107.733931:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.733935:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00080000:2.0:1734018107.733939:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.733942:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018107.733952:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018107.733962:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335467264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018107.733972:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.733976:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018107.733977:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.733981:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00080000:2.0:1734018107.733983:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.733987:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018107.733991:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018107.733999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335467392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018107.734008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.734011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018107.734013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.734016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00080000:2.0:1734018107.734018:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.734022:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018107.734026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018107.734031:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335467136:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018107.734035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335467520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018107.734044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.734047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018107.734049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.734052:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00080000:2.0:1734018107.734055:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.734058:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018107.734062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018107.734069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335467648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018107.734077:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.734080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018107.734082:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.734085:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00080000:2.0:1734018107.734088:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.734092:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018107.734096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734018107.734100:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018107.734103:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335467776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018107.734111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018107.734114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018107.734115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018107.734119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2372 00000100:00100000:0.0:1734018107.734121:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86de680 x1818247335467136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018107.734121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018107.734125:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018107.734128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018107.734136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335467904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018107.734150:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335467136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734161:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734164:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734168:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335467136:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018107.734175:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335467264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018107.734192:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018107.734199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86dc700 x1818247335467264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018107.734209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335467264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734217:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734220:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734223:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335467264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018107.734229:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335467392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018107.734242:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018107.734248:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86df100 x1818247335467392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018107.734258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335467392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734266:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734269:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335467392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018107.734299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335467520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018107.734312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018107.734318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86de300 x1818247335467520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018107.734329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335467520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335467520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018107.734346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018107.734358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018107.734364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86dce00 x1818247335467648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018107.734373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335467648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018107.734392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018107.734406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018107.734412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86dea00 x1818247335467776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018107.734423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335467776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018107.734440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018107.734452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018107.734457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018107/real 1734018107] req@ffff8800a86dc380 x1818247335467904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018107.734467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335467904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018107.734475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018107.734477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018107.734480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335467904:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018108.187976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018108.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018109.189949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018109.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018110.191943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018110.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018111.193909:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018111.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018112.195977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018112.213786:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018112.741839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.741848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018112.741853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.741863:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00080000:2.0:1734018112.741871:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.741879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018112.741915:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335468032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018112.741948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.741952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018112.741954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.741958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00080000:2.0:1734018112.741962:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.741966:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018112.741975:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018112.741985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335468160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018112.741995:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.741998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018112.742000:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.742003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00080000:2.0:1734018112.742006:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.742010:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018112.742014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018112.742022:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335468288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018112.742031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.742033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018112.742035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.742039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00080000:2.0:1734018112.742041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.742045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018112.742049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018112.742056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335468032:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018112.742059:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335468416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018112.742068:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.742071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018112.742072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.742076:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00080000:2.0:1734018112.742078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.742081:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018112.742086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018112.742093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335468544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018112.742102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.742105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018112.742106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.742110:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00080000:2.0:1734018112.742112:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.742115:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018112.742119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734018112.742123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018112.742126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335468672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018112.742135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018112.742138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018112.742140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018112.742143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2377 00000100:00100000:0.0:1734018112.742144:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86df800 x1818247335468032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018112.742146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018112.742149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018112.742153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018112.742160:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335468800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018112.742172:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335468032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742182:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742185:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742190:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335468032:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018112.742197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335468160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018112.742214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018112.742221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86dfb80 x1818247335468160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018112.742231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335468160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335468160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018112.742251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335468288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018112.742264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018112.742270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86ddf80 x1818247335468288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018112.742279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335468288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335468288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018112.742319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335468416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018112.742331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018112.742337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86dc380 x1818247335468416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018112.742348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335468416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742355:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335468416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018112.742366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335468544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018112.742377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018112.742383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86dea00 x1818247335468544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018112.742392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335468544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335468544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018112.742411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335468672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018112.742426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018112.742432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86dce00 x1818247335468672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018112.742442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335468672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742449:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335468672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018112.742460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335468800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018112.742471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018112.742477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018112/real 1734018112] req@ffff8800a86de300 x1818247335468800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018112.742487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335468800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018112.742494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018112.742496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018112.742499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335468800:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018113.198009:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018113.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018114.200000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018114.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018115.201997:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018115.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018116.203954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018116.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018117.205930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018117.213793:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734018117.749831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.749845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018117.749851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.749866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.749876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.749885:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018117.749934:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335468928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018117.749973:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.749977:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018117.749980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.749985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.749988:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.749992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018117.750005:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018117.750016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335469056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018117.750026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.750029:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018117.750031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.750036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.750038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.750042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018117.750047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018117.750055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335469184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018117.750064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.750067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018117.750068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.750072:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.750075:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.750078:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018117.750082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018117.750093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335469312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018117.750104:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335468928:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018117.750104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.750107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018117.750109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.750113:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.750116:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.750120:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018117.750124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018117.750132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335469440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018117.750140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.750143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018117.750145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.750148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.750151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.750154:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018117.750159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018117.750166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335469568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018117.750175:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018117.750178:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1734018117.750180:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018117.750180:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018117.750184:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2382 00000100:00080000:2.0:1734018117.750188:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018117.750192:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018117.750196:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018117.750202:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86df100 x1818247335468928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018117.750205:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335469696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018117.750230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335468928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335468928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018117.750258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018117.750275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018117.750281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86dc700 x1818247335469056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018117.750292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335469056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018117.750312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018117.750326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018117.750332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86de680 x1818247335469184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018117.750342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335469184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750349:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018117.750384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018117.750397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018117.750404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86dd500 x1818247335469312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018117.750414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335469312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750421:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018117.750433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335469440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018117.750445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018117.750494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86dd180 x1818247335469440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018117.750507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335469440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750517:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335469440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018117.750526:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335469568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018117.750545:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018117.750552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86dc000 x1818247335469568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018117.750563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335469568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750575:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335469568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018117.750581:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335469696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018117.750593:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018117.750599:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018117/real 1734018117] req@ffff8800a86ddc00 x1818247335469696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018117.750609:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335469696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018117.750617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018117.750619:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018117.750622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335469696:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018118.207985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018118.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018119.210014:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018119.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018120.211969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018120.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734018121.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018121.213963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018122.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018122.215948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018122.757809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.757823:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018122.757829:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.757842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.757854:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.757864:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018122.757911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335469824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018122.757948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.757952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018122.757955:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.757959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.757963:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.757967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018122.757978:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018122.757990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335469952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018122.758000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.758004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018122.758006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.758010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.758013:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.758016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018122.758021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018122.758030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335470080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018122.758038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.758041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018122.758042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.758046:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.758048:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.758052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018122.758056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018122.758062:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335469824:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018122.758067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335470208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018122.758076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.758079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018122.758081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.758086:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.758088:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.758092:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018122.758096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018122.758105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335470336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018122.758112:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.758115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018122.758117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.758121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.758123:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.758126:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018122.758130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018122.758138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335470464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018122.758145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018122.758148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018122.758150:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018122.758153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2387 00000100:00080000:2.0:1734018122.758156:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018122.758160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018122.758164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018122.758172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335470592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018122.758182:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018122.758205:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018122.758232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018122.758249:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335470336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018122.758264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335470464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018122.758279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335470592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018123.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018123.217987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018124.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018124.219954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018125.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018125.221979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018125.763991:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018125.764037:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018125.764056:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018125.764091:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018125.764099:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018125.764106:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018125.764113:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018125.764120:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018125.764127:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:2.0:1734018125.764211:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86ddc00 x1818247335470208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335470208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764266:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764272:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018125.764290:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86dc000 x1818247335470336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335470336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764395:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335470336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018125.764405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86dd180 x1818247335470464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764424:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335470464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335470464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018125.764457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86dd500 x1818247335470592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764475:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335470592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335470592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018125.764507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86df480 x1818247335469824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335469824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764543:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335469824:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018125.764559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86dd880 x1818247335469952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764587:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335469952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764599:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764610:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335469952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018125.764619:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018122/real 1734018125] req@ffff8800a86dca80 x1818247335470080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018125.764638:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335470080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018125.764650:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018125.764654:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018125.764659:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018126.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018126.223943:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018127.213949:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018127.225866:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018127.765829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.765840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018127.765844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018127.765855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.765864:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.765872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018127.765908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335470720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018127.765943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.765947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018127.765949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018127.765954:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.765957:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.765960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018127.765970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018127.765981:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335470848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018127.765991:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.765994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018127.765996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018127.765999:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.766002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.766005:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018127.766009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018127.766017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335470976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018127.766025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.766028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018127.766030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018127.766033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.766036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.766039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018127.766043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018127.766050:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335471104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018127.766057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335470720:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018127.766059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.766062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018127.766064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018127.766068:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.766070:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.766073:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018127.766077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018127.766084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335471232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018127.766092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.766095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018127.766096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018127.766100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.766102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.766105:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018127.766109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018127.766116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335471360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018127.766123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018127.766126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018127.766128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1734018127.766130:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018127.766131:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2392 00000100:00080000:2.0:1734018127.766134:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018127.766137:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018127.766140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018127.766148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335471488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018127.766151:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86dca80 x1818247335470720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766171:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335470720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766181:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766185:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766189:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335470720:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018127.766196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018127.766212:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018127.766219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86dd880 x1818247335470848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335470848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766240:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018127.766250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018127.766264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018127.766270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86df480 x1818247335470976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335470976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335470976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018127.766363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335471104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018127.766379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018127.766386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86dd500 x1818247335471104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335471104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335471104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018127.766416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335471232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018127.766427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018127.766434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86dd180 x1818247335471232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335471232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335471232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018127.766462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335471360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018127.766473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018127.766479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86dc000 x1818247335471360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335471360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335471360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018127.766506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335471488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018127.766518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018127.766524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018127/real 1734018127] req@ffff8800a86ddc00 x1818247335471488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018127.766534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335471488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018127.766548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018127.766551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018127.766554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335471488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018128.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018128.227989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018129.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018129.229920:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018130.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018130.231920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018131.213764:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018131.233818:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018132.213962:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018132.235885:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018132.773836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.773845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018132.773849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018132.773860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.773869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.773877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018132.773912:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335471616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018132.773945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.773950:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018132.773951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018132.773956:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.773959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.773962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018132.773972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018132.773983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335471744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018132.773992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.773995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018132.773997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018132.774001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.774003:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.774007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018132.774012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018132.774019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335471872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018132.774028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.774031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018132.774032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018132.774036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.774038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.774041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018132.774045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018132.774048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335471616:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018132.774055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335472000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018132.774063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.774066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018132.774068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018132.774071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.774074:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.774077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018132.774081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018132.774088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335472128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018132.774096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.774100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018132.774101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018132.774105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.774108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.774111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018132.774115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734018132.774117:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018132.774122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335472256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018132.774130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018132.774133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018132.774134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734018132.774137:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86de680 x1818247335471616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018132.774138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2397 00000100:00080000:2.0:1734018132.774140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018132.774143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018132.774147:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018132.774154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335472384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018132.774157:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335471616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774167:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774171:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774175:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335471616:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018132.774182:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335471744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018132.774247:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018132.774257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86dc700 x1818247335471744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018132.774270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335471744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335471744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018132.774297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335471872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018132.774315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018132.774321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86df100 x1818247335471872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018132.774332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335471872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335471872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018132.774372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018132.774385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018132.774391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86de300 x1818247335472000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018132.774401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335472000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018132.774421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335472128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018132.774433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018132.774439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86dce00 x1818247335472128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018132.774449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335472128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335472128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018132.774467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335472256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018132.774479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018132.774484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86dea00 x1818247335472256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018132.774494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335472256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335472256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018132.774512:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335472384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018132.774524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018132.774530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018132/real 1734018132] req@ffff8800a86dc380 x1818247335472384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018132.774540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335472384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018132.774548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018132.774550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018132.774553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335472384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018133.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018133.237859:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018134.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018134.239929:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018135.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018135.241933:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018136.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018136.243924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018137.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018137.245988:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018137.781809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018137.781818:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018137.781822:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018137.781832:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.781841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.781849:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018137.781885:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335472512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018137.781918:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018137.781921:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018137.781923:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018137.781928:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.781931:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.781935:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018137.781944:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018137.781955:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335472640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018137.781964:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018137.781968:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018137.781969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018137.781973:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.781976:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.781979:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018137.781984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018137.781991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335472768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018137.782000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018137.782003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018137.782005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018137.782009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.782011:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.782015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018137.782018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018137.782028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335472896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018137.782037:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734018137.782040:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335472512:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018137.782040:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018137.782042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018137.782045:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.782048:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.782051:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018137.782055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018137.782063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335473024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018137.782072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018137.782074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018137.782076:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018137.782079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.782081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.782085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018137.782089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018137.782096:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335473152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018137.782104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018137.782107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018137.782108:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1734018137.782136:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018137.782142:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2402 00000100:00080000:2.0:1734018137.782148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018137.782153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782158:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86ddf80 x1818247335472512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734018137.782158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018137.782168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335473280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018137.782177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335472512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782188:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782192:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782197:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335472512:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018137.782204:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018137.782224:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782231:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86dfb80 x1818247335472640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018137.782242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335472640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018137.782263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018137.782277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86df800 x1818247335472768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018137.782293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335472768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018137.782333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018137.782345:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86dc380 x1818247335472896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018137.782362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335472896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335472896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018137.782381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018137.782395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86dea00 x1818247335473024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018137.782412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335473024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782421:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018137.782430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018137.782441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86dce00 x1818247335473152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018137.782457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335473152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018137.782475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018137.782487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018137.782493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018137/real 1734018137] req@ffff8800a86de300 x1818247335473280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018137.782509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335473280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018137.782517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018137.782519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018137.782522:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018138.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018138.247964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018139.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018139.249957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018140.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018140.251942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018141.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018141.253961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018142.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018142.256001:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018142.789842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.789854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018142.789859:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018142.789872:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2407 00000100:00080000:2.0:1734018142.789883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.789893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018142.789942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335473408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018142.789988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.789994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018142.789996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018142.790005:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2407 00000100:00080000:2.0:1734018142.790010:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.790016:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018142.790076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018142.790084:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335473408:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018142.790095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335473536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018142.790111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.790115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018142.790117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018142.790123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2407 00000100:00080000:2.0:1734018142.790127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.790133:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018142.790140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018142.790152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335473664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018142.790164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.790168:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018142.790171:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018142.790176:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2407 00000800:00000100:0.0:1734018142.790178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018142.790180:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.790186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018142.790192:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018142.790205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86df100 x1818247335473408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018142.790207:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335473792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018142.790222:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.790227:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018142.790229:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734018142.790231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335473408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018142.790235:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2407 00000100:00080000:2.0:1734018142.790239:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.790244:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:0.0:1734018142.790246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734018142.790251:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018142.790258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335473408:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018142.790264:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335473920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018142.790276:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.790281:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018142.790283:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018142.790289:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2407 00000100:00100000:0.0:1734018142.790292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335473536:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734018142.790293:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.790298:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018142.790305:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734018142.790319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018142.790319:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335474048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018142.790330:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86dc700 x1818247335473536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018142.790334:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018142.790340:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018142.790343:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1734018142.790348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335473536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018142.790349:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2407 00000100:00080000:2.0:1734018142.790354:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018142.790359:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:0.0:1734018142.790361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734018142.790366:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018142.790372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335473536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018142.790378:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335474176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018142.790382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335473664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018142.790407:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018142.790418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86de680 x1818247335473664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018142.790436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335473664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018142.790449:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018142.790458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335473664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018142.790468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335473792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018142.790494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018142.790504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86ddc00 x1818247335473792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018142.790515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335473792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018142.790524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018142.790529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335473792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018142.790534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018142.790554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018142.790560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86dc000 x1818247335473920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018142.790571:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335473920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018142.790579:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018142.790584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335473920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018142.790604:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018142.790616:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018142.790622:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86dd180 x1818247335474048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018142.790632:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335474048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018142.790640:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790642:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018142.790646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018142.790651:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018142.790663:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018142.790669:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018142/real 1734018142] req@ffff8800a86dd500 x1818247335474176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018142.790686:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335474176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018142.790738:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018142.790741:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018142.790745:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018143.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018143.257986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018144.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018144.259989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018145.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018145.261966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018146.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018146.264012:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018147.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018147.266004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018147.797843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018147.797852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018147.797857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.797867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.797876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.797884:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018147.797921:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335474304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018147.797998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018147.798003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018147.798005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.798011:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.798015:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.798018:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018147.798029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018147.798041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335474432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018147.798051:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018147.798054:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018147.798056:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.798059:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.798062:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.798065:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018147.798069:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018147.798077:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335474560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018147.798086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018147.798089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018147.798090:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.798094:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.798096:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.798099:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018147.798103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018147.798109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335474304:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018147.798113:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335474688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018147.798128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018147.798132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018147.798133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.798137:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.798139:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.798142:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018147.798146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018147.798154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335474816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018147.798162:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018147.798164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018147.798166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.798169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.798172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.798175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:0.0:1734018147.798177:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734018147.798179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018147.798187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335474944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018147.798195:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1734018147.798198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86df480 x1818247335474304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018147.798198:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018147.798199:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018147.798203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2412 00000100:00080000:2.0:1734018147.798206:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018147.798209:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018147.798213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018147.798218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335474304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018147.798220:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335475072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018147.798229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798237:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335474304:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018147.798244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335474432:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018147.798260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018147.798267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86dd880 x1818247335474432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018147.798278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335474432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018147.798286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335474432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018147.798299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335474560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018147.798313:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018147.798318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86dca80 x1818247335474560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018147.798328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335474560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018147.798336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335474560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018147.798368:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335474688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018147.798380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018147.798386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86dd500 x1818247335474688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018147.798397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335474688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018147.798405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335474688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018147.798416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018147.798431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018147.798437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86dd180 x1818247335474816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018147.798448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335474816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018147.798455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018147.798465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018147.798476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018147.798482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86dc000 x1818247335474944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018147.798492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335474944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018147.798500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335474944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018147.798511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018147.798522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018147.798528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018147/real 1734018147] req@ffff8800a86ddc00 x1818247335475072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018147.798544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335475072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018147.798552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018147.798554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018147.798557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018148.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018148.267927:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018149.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018149.269996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018150.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018150.271970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018151.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018151.273972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018152.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018152.275966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018152.805848:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.805899:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018152.805904:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.805914:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.805923:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018152.805931:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018152.805968:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335475200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018152.806003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.806007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018152.806009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.806014:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.806017:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018152.806021:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018152.806030:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018152.806042:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335475328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018152.806052:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.806055:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018152.806057:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.806060:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.806063:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018152.806066:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018152.806070:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018152.806078:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335475456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018152.806088:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.806091:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018152.806093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.806096:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.806099:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734018152.806101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335475200:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018152.806102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018152.806106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018152.806116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335475584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018152.806124:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.806127:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018152.806129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.806133:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.806135:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018152.806138:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018152.806143:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018152.806150:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335475712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018152.806158:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.806161:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018152.806162:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.806165:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.806167:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018152.806171:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018152.806174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018152.806181:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335475840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018152.806189:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018152.806192:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018152.806193:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018152.806196:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2417 00000100:00080000:2.0:1734018152.806199:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018152.806202:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:0.0:1734018152.806203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734018152.806206:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018152.806214:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335475968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018152.806230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86de680 x1818247335475200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335475200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806266:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335475200:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018152.806285:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335475328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018152.806304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018152.806312:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86dc700 x1818247335475328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335475328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806331:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335475328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018152.806369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335475456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018152.806385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018152.806391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86df100 x1818247335475456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335475456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335475456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018152.806422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335475584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018152.806434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018152.806440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86de300 x1818247335475584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335475584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335475584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018152.806470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018152.806487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018152.806494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86dce00 x1818247335475712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335475712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018152.806525:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018152.806537:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018152.806543:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86dea00 x1818247335475840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806553:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335475840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018152.806572:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018152.806588:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018152.806594:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018152/real 1734018152] req@ffff8800a86dc380 x1818247335475968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018152.806612:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335475968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018152.806620:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018152.806622:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018152.806625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335475968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018153.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018153.278001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018154.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018154.279985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018155.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018155.282006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018156.213889:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018156.284028:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018157.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018157.286005:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018157.813866:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.813877:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018157.813882:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.813893:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.813903:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.813911:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018157.813948:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335476096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018157.813982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.813987:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018157.813989:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.813993:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.813996:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.814000:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018157.814010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018157.814020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335476224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018157.814030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.814033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018157.814035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.814039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.814042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.814045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018157.814050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018157.814057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335476352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018157.814066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.814069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018157.814070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.814074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.814076:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.814079:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018157.814083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018157.814093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335476480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018157.814101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335476096:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018157.814102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.814105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018157.814106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.814110:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.814113:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.814116:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018157.814120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018157.814128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335476608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018157.814137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.814140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018157.814141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.814144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.814147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.814150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018157.814154:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018157.814161:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335476736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018157.814169:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018157.814171:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1734018157.814173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018157.814173:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018157.814176:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2422 00000100:00080000:2.0:1734018157.814179:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018157.814182:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018157.814186:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018157.814193:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335476864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018157.814194:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86df800 x1818247335476096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335476096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335476096:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018157.814240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335476224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018157.814256:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018157.814263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86dfb80 x1818247335476224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335476224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814285:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335476224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018157.814295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335476352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018157.814309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018157.814315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86ddf80 x1818247335476352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335476352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814338:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335476352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018157.814365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335476480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018157.814377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018157.814384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86dc380 x1818247335476480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335476480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335476480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018157.814413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335476608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018157.814428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018157.814434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86dea00 x1818247335476608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335476608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335476608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018157.814462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335476736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018157.814473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018157.814479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86dce00 x1818247335476736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335476736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335476736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018157.814506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335476864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018157.814518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018157.814524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018157/real 1734018157] req@ffff8800a86de300 x1818247335476864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018157.814540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335476864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018157.814548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018157.814550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018157.814553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335476864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018158.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018158.287984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018159.213873:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018159.289969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018160.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018160.291993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018161.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018161.293959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018162.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018162.295939:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018162.821836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.821850:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018162.821856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.821870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.821883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.821893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018162.821941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335476992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018162.821978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.821982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018162.821984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.821989:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.821993:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.821996:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018162.822008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018162.822019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335477120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018162.822029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.822032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018162.822035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.822039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.822041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.822045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018162.822050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018162.822058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335477248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018162.822068:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.822071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018162.822073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.822076:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.822079:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.822082:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018162.822087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018162.822089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335476992:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018162.822098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335477376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018162.822107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.822110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018162.822112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.822115:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.822118:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.822121:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018162.822126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018162.822134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335477504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018162.822142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.822145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018162.822147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.822150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.822154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.822157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018162.822162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018162.822169:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335477632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018162.822174:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018162.822177:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018162.822180:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018162.822182:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018162.822185:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2427 00000100:00080000:2.0:1734018162.822188:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018162.822191:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018162.822195:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018162.822199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86df100 x1818247335476992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018162.822204:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335477760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018162.822220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335476992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335476992:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018162.822247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335477120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018162.822264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018162.822271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86dc700 x1818247335477120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018162.822282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335477120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822298:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335477120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018162.822304:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335477248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018162.822319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018162.822325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86de680 x1818247335477248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018162.822335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335477248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335477248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018162.822381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335477376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018162.822393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018162.822399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86ddc00 x1818247335477376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018162.822410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335477376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822421:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335477376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018162.822429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335477504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018162.822445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018162.822451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86dc000 x1818247335477504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018162.822461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335477504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335477504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018162.822478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335477632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018162.822489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018162.822495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86dd180 x1818247335477632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018162.822505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335477632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335477632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018162.822523:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335477760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018162.822536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018162.822542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018162/real 1734018162] req@ffff8800a86dd500 x1818247335477760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018162.822559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335477760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018162.822567:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018162.822569:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018162.822572:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335477760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018163.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018163.297946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018164.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018164.299982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018165.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018165.301988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018166.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018166.303927:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018167.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018167.306064:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018167.829821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.829832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018167.829837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.829848:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.829858:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018167.829867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018167.829906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335477888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018167.829941:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.829945:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018167.829947:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.829952:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.829955:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018167.829959:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018167.829969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018167.829979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335478016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018167.829989:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.829993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018167.829994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.829998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.830000:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018167.830004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018167.830008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018167.830015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335478144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018167.830024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.830027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018167.830029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.830033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.830035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018167.830039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018167.830043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018167.830053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335478272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018167.830063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.830066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018167.830067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.830071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.830074:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1734018167.830075:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335477888:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734018167.830077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018167.830081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018167.830089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335478400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018167.830097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.830100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018167.830101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.830105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.830107:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018167.830111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018167.830115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018167.830122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335478528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018167.830131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018167.830134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018167.830136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018167.830139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2432 00000100:00080000:2.0:1734018167.830142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018167.830145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018167.830149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018167.830156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335478656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018167.830167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830193:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86dca80 x1818247335477888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335477888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335477888:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018167.830246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018167.830265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86dd880 x1818247335478016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335478016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018167.830306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018167.830321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86df480 x1818247335478144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335478144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018167.830382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018167.830395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86dd500 x1818247335478272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335478272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830421:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830423:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830426:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018167.830432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335478400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018167.830448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86dd180 x1818247335478400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335478400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335478400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018167.830483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335478528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018167.830496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86dc000 x1818247335478528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335478528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335478528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018167.830530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335478656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018167.830543:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018167.830548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018167/real 1734018167] req@ffff8800a86ddc00 x1818247335478656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018167.830565:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335478656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018167.830574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018167.830576:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018167.830579:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335478656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018168.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018168.307976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018169.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018169.309996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018170.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018170.312029:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018171.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018171.313973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018172.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018172.315992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018172.837860:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.837873:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018172.837879:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.837892:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.837903:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.837913:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018172.837960:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335478784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018172.837996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.838001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018172.838003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.838008:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.838012:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.838016:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018172.838027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018172.838038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335478912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018172.838048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.838052:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018172.838054:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.838058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.838062:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.838065:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018172.838070:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018172.838078:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335479040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018172.838087:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.838090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018172.838091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.838095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.838098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.838102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018172.838106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018172.838109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335478784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018172.838118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335479168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018172.838128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.838131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018172.838133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.838136:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.838139:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.838142:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018172.838146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018172.838154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335479296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018172.838162:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.838165:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018172.838166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.838170:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.838172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.838176:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018172.838180:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018172.838187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335479424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:0.0:1734018172.838195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734018172.838195:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018172.838198:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018172.838199:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018172.838203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2437 00000100:00080000:2.0:1734018172.838207:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018172.838210:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018172.838214:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018172.838221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86de680 x1818247335478784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018172.838223:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335479552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018172.838250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335478784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838272:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838279:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335478784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018172.838292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478912:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018172.838325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018172.838339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86dc700 x1818247335478912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018172.838355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335478912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335478912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018172.838379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018172.838400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018172.838406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86df100 x1818247335479040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018172.838418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335479040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838425:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838427:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018172.838463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018172.838476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018172.838482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86de300 x1818247335479168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018172.838492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335479168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018172.838511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335479296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018172.838578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018172.838586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86dce00 x1818247335479296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018172.838597:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335479296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838606:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838608:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838611:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335479296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018172.838616:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335479424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018172.838628:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018172.838634:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86dea00 x1818247335479424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018172.838643:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335479424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838652:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838654:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838657:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335479424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018172.838662:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335479552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018172.838675:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018172.838680:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018172/real 1734018172] req@ffff8800a86dc380 x1818247335479552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018172.838739:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335479552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018172.838748:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018172.838750:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018172.838753:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335479552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018173.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018173.317970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018174.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018174.319996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018175.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018175.321975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018176.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018176.324002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018177.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018177.325979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734018177.845833:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.845842:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734018177.845847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.845857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00080000:2.0:1734018177.845866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.845874:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734018177.845911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335479680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018177.845944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.845949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018177.845951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.845955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00080000:2.0:1734018177.845958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.845962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018177.845972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018177.845983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335479808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018177.845993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.845996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018177.845998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.846001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00080000:2.0:1734018177.846004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.846007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018177.846011:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734018177.846019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335479936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018177.846027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.846030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018177.846032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.846035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00080000:2.0:1734018177.846038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.846041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018177.846045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018177.846051:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335479680:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734018177.846055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335480064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018177.846064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.846067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734018177.846068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.846072:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00080000:2.0:1734018177.846075:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.846078:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018177.846082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734018177.846090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335480192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018177.846097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.846100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018177.846102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.846105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00080000:2.0:1734018177.846109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.846112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018177.846116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1734018177.846120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018177.846123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335480320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734018177.846131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734018177.846134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734018177.846136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018177.846139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2442 00000100:00100000:0.0:1734018177.846141:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86ddf80 x1818247335479680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018177.846142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734018177.846146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734018177.846150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734018177.846157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335480448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:0.0:1734018177.846160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335479680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846171:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335479680:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018177.846186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018177.846203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018177.846209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86dfb80 x1818247335479808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018177.846220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335479808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846232:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018177.846241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018177.846255:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018177.846261:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86df800 x1818247335479936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018177.846271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335479936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335479936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018177.846312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018177.846325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018177.846331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86dc380 x1818247335480064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018177.846342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335480064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018177.846360:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335480192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:0.0:1734018177.846375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018177.846381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86dea00 x1818247335480192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018177.846392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335480192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846404:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335480192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018177.846410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335480320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018177.846421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018177.846475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86dce00 x1818247335480320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018177.846489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335480320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335480320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018177.846508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335480448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:0.0:1734018177.846525:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018177.846532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018177/real 1734018177] req@ffff8800a86de300 x1818247335480448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018177.846549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335480448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018177.846557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018177.846559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018177.846562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335480448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018178.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018178.327998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018179.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018179.329983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018180.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018180.331948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018181.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018181.333931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018182.213873:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018182.335991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018182.853911:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.853919:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018182.853924:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018182.853934:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.853943:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018182.853952:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018182.853988:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335480576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018182.854028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.854032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018182.854034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018182.854039:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.854042:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018182.854046:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018182.854055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018182.854067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335480704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018182.854072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335480576:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018182.854084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.854087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018182.854089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018182.854093:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.854095:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018182.854098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018182.854103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018182.854112:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335480832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018182.854120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.854123:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018182.854124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018182.854128:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.854131:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018182.854134:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018182.854138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734018182.854140:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018182.854147:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335480960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018182.854155:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.854158:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018182.854160:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1734018182.854162:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59c9500 x1818247335480576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018182.854163:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.854167:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018182.854170:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018182.854174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018182.854182:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335481088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018182.854182:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335480576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018182.854189:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.854192:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734018182.854193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018182.854194:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734018182.854196:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734018182.854197:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.854200:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1734018182.854201:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335480576:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018182.854203:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018182.854207:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018182.854214:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335481216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018182.854222:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018182.854225:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1734018182.854226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480704:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734018182.854227:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018182.854230:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2447 00000100:00080000:0.0:1734018182.854233:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018182.854236:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018182.854239:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1734018182.854244:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018182.854247:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335481344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018182.854252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59c8000 x1818247335480704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018182.854263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335480704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018182.854271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018182.854274:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018182.854277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018182.854284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018182.854298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018182.854304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59ca300 x1818247335480832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018182.854315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335480832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018182.854323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018182.854325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018182.854328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018182.854333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018182.854377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018182.854385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59c9180 x1818247335480960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018182.854395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335480960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018182.854403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018182.854405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018182.854408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335480960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018182.854414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:2.0:1734018182.854429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018182.854435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59cad80 x1818247335481088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018182.854446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335481088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018182.854454:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018182.854456:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018182.854459:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018182.854465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018182.854477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018182.854483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59cb800 x1818247335481216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018182.854492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335481216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018182.854500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018182.854502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018182.854505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018182.854527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:2.0:1734018182.854539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734018182.854545:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018182/real 1734018182] req@ffff8800b59cb100 x1818247335481344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734018182.854556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335481344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734018182.854563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734018182.854565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1734018182.854568:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018183.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018183.337963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018184.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018184.339953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018185.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018185.342007:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018186.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018186.344008:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018187.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018187.345934:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018187.861838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018187.861852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018187.861857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.861872:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.861884:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.861894:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018187.861941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335481472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018187.861981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018187.861986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018187.861988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.861993:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.861996:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.862000:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018187.862012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018187.862023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335481600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018187.862033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018187.862037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018187.862039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.862043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.862045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.862049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018187.862055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018187.862064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335481728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018187.862073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018187.862076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018187.862078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.862082:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.862084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.862087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018187.862092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018187.862100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335481856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018187.862109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1734018187.862109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335481472:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018187.862113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018187.862114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.862118:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.862121:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.862125:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018187.862129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018187.862137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335481984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018187.862145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018187.862149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018187.862151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.862155:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.862158:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.862161:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018187.862166:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018187.862173:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335482112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018187.862181:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018187.862184:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018187.862185:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018187.862188:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2452 00000100:00080000:0.0:1734018187.862192:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018187.862195:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018187.862199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018187.862210:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335482240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018187.862210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335481600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018187.862234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335481728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018187.862288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335481856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018187.862307:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018187.862322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335482112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018187.862337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335482240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018188.213938:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018188.347958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018189.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018189.349980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018190.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018190.351911:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018190.867982:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018190.868028:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018190.868048:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018190.868084:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018190.868092:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018190.868100:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018190.868107:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018190.868114:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018190.868121:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018190.868251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59c8700 x1818247335481856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335481856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868317:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335481856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018190.868337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59c9f80 x1818247335481984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335481984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335481984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018190.868391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59cb480 x1818247335482112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335482112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868432:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335482112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018190.868453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59cbb80 x1818247335482240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335482240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335482240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018190.868503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59ca680 x1818247335481472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335481472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335481472:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018190.868554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59c8e00 x1818247335481600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335481600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868595:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868604:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335481600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018190.868614:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018187/real 1734018190] req@ffff8800b59c8a80 x1818247335481728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018190.868631:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335481728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018190.868644:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018190.868647:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018190.868653:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335481728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018191.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018191.353969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018192.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018192.355995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018192.869853:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.869864:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018192.869870:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018192.869883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.869894:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.869904:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018192.869952:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335482368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018192.869992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.869997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018192.869999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018192.870004:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.870007:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.870011:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018192.870023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018192.870035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335482496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018192.870045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.870048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018192.870050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018192.870054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.870056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.870060:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018192.870065:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018192.870076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335482624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018192.870084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.870090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018192.870092:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018192.870096:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.870099:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.870102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018192.870107:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018192.870113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335482368:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018192.870115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335482752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018192.870125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.870128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018192.870130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018192.870134:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.870137:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.870140:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018192.870145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018192.870152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335482880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018192.870160:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.870163:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018192.870164:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018192.870168:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.870171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.870204:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018192.870209:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018192.870220:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335483008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018192.870229:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018192.870232:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018192.870234:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734018192.870236:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018192.870238:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2457 00000100:00080000:0.0:1734018192.870241:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018192.870245:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018192.870249:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018192.870258:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335483136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018192.870262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59c9c00 x1818247335482368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335482368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870306:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335482368:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018192.870315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335482496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018192.870335:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018192.870342:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59caa00 x1818247335482496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335482496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335482496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018192.870374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335482624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018192.870389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018192.870395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59c8a80 x1818247335482624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335482624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870418:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335482624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018192.870447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335482752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018192.870464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018192.870470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59c8e00 x1818247335482752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335482752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335482752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018192.870500:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335482880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018192.870512:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018192.870518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59ca680 x1818247335482880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335482880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335482880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018192.870546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018192.870558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018192.870564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59cbb80 x1818247335483008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870581:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335483008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870589:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870591:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870594:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018192.870599:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018192.870612:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018192.870617:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018192/real 1734018192] req@ffff8800b59cb480 x1818247335483136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018192.870627:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335483136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018192.870634:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018192.870637:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018192.870639:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018193.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018193.357964:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018194.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018194.359936:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018195.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018195.361957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018196.213919:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018196.363915:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018197.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018197.366010:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018197.877819:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.877832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018197.877838:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.877852:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.877863:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.877873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018197.877921:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335483264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018197.877961:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.877969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018197.877971:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.877976:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.877979:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.877983:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018197.877995:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018197.878007:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335483392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018197.878017:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.878020:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018197.878022:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.878026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.878029:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.878032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018197.878037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018197.878045:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335483520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018197.878054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.878057:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018197.878059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.878063:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.878065:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.878069:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018197.878074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018197.878082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335483648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018197.878116:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335483264:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018197.878121:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.878125:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018197.878127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.878132:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.878135:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.878139:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018197.878144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018197.878153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335483776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018197.878162:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.878166:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018197.878167:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.878171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.878174:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.878177:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018197.878181:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018197.878190:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335483904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018197.878191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018197.878197:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018197.878200:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018197.878203:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018197.878207:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2462 00000100:00080000:0.0:1734018197.878210:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018197.878213:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59c9f80 x1818247335483264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018197.878217:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018197.878227:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335484032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018197.878237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335483264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335483264:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018197.878266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335483392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018197.878285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59c8700 x1818247335483392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018197.878304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335483392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335483392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018197.878324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335483520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018197.878339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59cb100 x1818247335483520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018197.878355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335483520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335483520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018197.878399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335483648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018197.878412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59cb800 x1818247335483648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018197.878429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335483648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335483648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018197.878448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018197.878460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59cad80 x1818247335483776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018197.878476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335483776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018197.878495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018197.878510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59c9180 x1818247335483904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018197.878527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335483904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335483904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018197.878545:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018197.878558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018197.878564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018197/real 1734018197] req@ffff8800b59ca300 x1818247335484032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018197.878573:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335484032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018197.878580:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018197.878583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018197.878586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018198.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018198.367982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018199.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018199.369953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018200.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018200.371943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018201.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018201.373941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018202.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018202.375987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018202.885839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.885847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018202.885852:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018202.885861:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.885870:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018202.885878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018202.885913:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335484160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018202.885947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.885952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018202.885954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018202.885958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.885961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018202.885964:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018202.885974:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018202.885985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335484288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018202.886026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.886030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018202.886031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018202.886036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.886038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018202.886042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018202.886047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018202.886057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335484416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018202.886067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.886070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018202.886072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018202.886075:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.886078:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018202.886080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335484160:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018202.886081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018202.886085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018202.886093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335484544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018202.886101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.886104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018202.886106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018202.886109:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.886112:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018202.886116:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018202.886120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018202.886127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335484672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018202.886135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.886137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018202.886139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018202.886142:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.886145:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018202.886148:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734018202.886148:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734018202.886152:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018202.886159:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335484800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018202.886167:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018202.886170:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018202.886171:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018202.886171:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59c8000 x1818247335484160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018202.886175:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2467 00000100:00080000:0.0:1734018202.886177:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018202.886180:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018202.886184:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018202.886192:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335484928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018202.886192:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335484160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886211:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335484160:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018202.886218:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335484288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018202.886235:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018202.886242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59c9500 x1818247335484288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018202.886253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335484288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335484288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018202.886291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335484416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018202.886304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018202.886311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59ca300 x1818247335484416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018202.886321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335484416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335484416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018202.886341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335484544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018202.886353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018202.886359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59c9180 x1818247335484544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018202.886369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335484544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335484544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018202.886387:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018202.886399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018202.886405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59cad80 x1818247335484672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018202.886415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335484672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018202.886435:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018202.886447:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018202.886453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59cb800 x1818247335484800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018202.886463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335484800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018202.886481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018202.886493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018202.886499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018202/real 1734018202] req@ffff8800b59cb100 x1818247335484928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018202.886509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335484928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018202.886516:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018202.886518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018202.886521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335484928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018203.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018203.377939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018204.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018204.379931:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018205.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018205.381931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018206.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018206.384037:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018207.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018207.386031:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018207.893846:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.893858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018207.893864:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.893878:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.893889:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018207.893899:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018207.893975:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335485056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018207.894011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.894016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018207.894018:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.894023:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.894026:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018207.894030:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018207.894042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018207.894054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335485184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018207.894065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.894068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018207.894070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.894074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.894077:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018207.894081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018207.894087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018207.894096:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335485312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018207.894105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.894108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018207.894110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.894114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.894116:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018207.894120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018207.894124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018207.894129:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335485056:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018207.894133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335485440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018207.894143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.894146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018207.894148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.894152:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.894154:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018207.894158:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018207.894162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018207.894171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335485568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018207.894179:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.894182:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018207.894184:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.894189:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.894192:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018207.894196:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018207.894200:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018207.894203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018207.894208:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335485696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018207.894216:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018207.894219:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018207.894221:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018207.894224:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2472 00000100:00080000:0.0:1734018207.894227:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018207.894229:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59c8700 x1818247335485056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018207.894231:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018207.894235:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018207.894245:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335485824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018207.894249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335485056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335485056:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018207.894279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335485184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018207.894298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018207.894305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59c9f80 x1818247335485184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018207.894317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335485184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335485184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018207.894338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335485312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018207.894352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018207.894358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59cb480 x1818247335485312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018207.894370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335485312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894378:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335485312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018207.894412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335485440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018207.894425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018207.894431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59cbb80 x1818247335485440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018207.894441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335485440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894449:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335485440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018207.894461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335485568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018207.894474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018207.894480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59ca680 x1818247335485568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018207.894489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335485568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335485568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018207.894510:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335485696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018207.894522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018207.894528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59c8e00 x1818247335485696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018207.894537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335485696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894545:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335485696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018207.894556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335485824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018207.894567:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018207.894573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018207/real 1734018207] req@ffff8800b59c8a80 x1818247335485824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018207.894583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335485824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018207.894591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018207.894593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018207.894596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335485824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018208.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018208.387994:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018209.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018209.389966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018210.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018210.392025:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018211.213897:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018211.394005:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018212.213902:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018212.396006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018212.901859:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018212.901871:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018212.901877:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.901892:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.901902:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.901911:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018212.901959:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335485952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018212.902005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018212.902010:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018212.902011:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.902017:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.902020:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.902024:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018212.902035:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018212.902048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335486080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018212.902058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018212.902062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018212.902063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.902067:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.902070:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.902074:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018212.902079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018212.902089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335486208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018212.902098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018212.902101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018212.902103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.902106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.902109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.902114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018212.902118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018212.902128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335486336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018212.902129:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335485952:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018212.902137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018212.902141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018212.902142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.902146:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.902148:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.902152:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018212.902157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018212.902165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335486464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018212.902172:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018212.902176:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018212.902178:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.902182:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.902184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.902188:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018212.902192:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018212.902200:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335486592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018212.902208:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734018212.902210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018212.902212:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018212.902214:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018212.902218:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2477 00000100:00080000:0.0:1734018212.902222:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018212.902225:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018212.902229:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018212.902235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59caa00 x1818247335485952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018212.902238:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335486720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018212.902258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335485952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335485952:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018212.902289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018212.902307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018212.902314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59c9c00 x1818247335486080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018212.902325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335486080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018212.902347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018212.902361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018212.902368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59c8a80 x1818247335486208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018212.902379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335486208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018212.902420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018212.902433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018212.902440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59c8e00 x1818247335486336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018212.902450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335486336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018212.902470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335486464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018212.902482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018212.902489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59ca680 x1818247335486464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018212.902499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335486464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902517:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335486464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018212.902525:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335486592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018212.902537:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018212.902543:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59cbb80 x1818247335486592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018212.902553:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335486592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335486592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018212.902571:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335486720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018212.902583:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018212.902588:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018212/real 1734018212] req@ffff8800b59cb480 x1818247335486720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018212.902599:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335486720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018212.902606:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018212.902609:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018212.902612:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335486720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018213.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018213.398010:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018214.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018214.399956:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018215.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018215.401989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018216.213892:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018216.403987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018217.213873:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018217.405988:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018217.909839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.909848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018217.909853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.909863:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00080000:0.0:1734018217.909873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.909881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018217.909916:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335486848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018217.909951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.909955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018217.909957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.909962:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00080000:0.0:1734018217.909965:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.909969:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018217.909979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018217.909989:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335486976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018217.910001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.910004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018217.910006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.910009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00080000:0.0:1734018217.910011:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.910015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018217.910019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018217.910027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335487104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018217.910034:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335486848:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018217.910036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.910039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018217.910041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.910044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00080000:0.0:1734018217.910047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.910050:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018217.910054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018217.910061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335487232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018217.910069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.910072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018217.910074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.910078:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00080000:0.0:1734018217.910081:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.910084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018217.910088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018217.910095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335487360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018217.910102:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018217.910103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.910106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018217.910108:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.910111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00080000:0.0:1734018217.910114:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.910117:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018217.910121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018217.910125:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59c9f80 x1818247335486848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018217.910128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335487488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018217.910136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018217.910139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018217.910140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018217.910144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2482 00000100:00100000:3.0:1734018217.910144:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335486848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018217.910146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018217.910149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018217.910153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734018217.910155:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910159:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018217.910162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335487616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018217.910164:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335486848:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018217.910171:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018217.910188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018217.910195:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59c8700 x1818247335486976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018217.910206:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335486976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018217.910214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018217.910220:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335486976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018217.910247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335487104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018217.910261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018217.910267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59cb100 x1818247335487104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018217.910278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335487104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018217.910285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018217.910290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335487104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018217.910296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335487232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018217.910307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018217.910313:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59cb800 x1818247335487232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018217.910323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335487232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018217.910330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018217.910335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335487232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018217.910341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335487360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018217.910353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018217.910359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59cad80 x1818247335487360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018217.910370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335487360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018217.910378:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018217.910383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335487360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018217.910388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335487488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018217.910400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018217.910406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59c9180 x1818247335487488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018217.910416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335487488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018217.910423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018217.910428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335487488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018217.910434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335487616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018217.910446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018217.910452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018217/real 1734018217] req@ffff8800b59ca300 x1818247335487616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018217.910462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335487616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018217.910469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018217.910472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018217.910475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335487616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018218.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018218.407971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018219.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018219.409921:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018220.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018220.411956:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018221.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018221.413942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018222.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018222.415972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018222.917840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.917853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018222.917858:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.917872:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.917884:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.917894:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018222.917943:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335487744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018222.917981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.917986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018222.917988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.917993:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.917996:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.918000:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018222.918012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018222.918023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335487872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018222.918033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.918037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018222.918038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.918043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.918045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.918049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018222.918055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018222.918067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335488000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018222.918076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.918079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018222.918081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.918085:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.918087:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.918091:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018222.918095:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018222.918103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335487744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018222.918104:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335488128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018222.918113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.918117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018222.918118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.918122:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.918125:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.918129:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018222.918134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018222.918142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335488256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018222.918150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.918153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018222.918156:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.918159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.918163:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.918166:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018222.918170:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018222.918178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335488384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018222.918179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018222.918187:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018222.918190:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018222.918191:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018222.918194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2487 00000100:00080000:0.0:1734018222.918197:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018222.918200:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018222.918204:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018222.918204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59c9500 x1818247335487744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018222.918213:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335488512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018222.918225:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335487744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918240:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335487744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018222.918253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335487872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018222.918271:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018222.918278:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59c8000 x1818247335487872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018222.918290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335487872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335487872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018222.918312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018222.918326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018222.918332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59ca300 x1818247335488000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018222.918342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335488000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018222.918384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018222.918397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018222.918403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59c9180 x1818247335488128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018222.918414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335488128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018222.918433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335488256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018222.918446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018222.918452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59cad80 x1818247335488256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018222.918463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335488256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335488256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018222.918481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335488384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018222.918494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018222.918500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59cb800 x1818247335488384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018222.918509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335488384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918522:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335488384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018222.918528:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335488512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018222.918540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018222.918546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018222/real 1734018222] req@ffff8800b59cb100 x1818247335488512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018222.918556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335488512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018222.918570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018222.918573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018222.918576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335488512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018223.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018223.417988:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018224.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018224.419992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018225.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018225.421980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018226.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018226.423961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018227.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018227.425951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018227.925806:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.925819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018227.925825:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018227.925839:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.925851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.925861:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018227.925910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335488640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018227.925964:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.925969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018227.925971:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018227.925976:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.925980:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.925983:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018227.925995:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018227.926008:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335488768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018227.926018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.926021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018227.926023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018227.926027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.926029:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.926033:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018227.926037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018227.926046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335488896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018227.926054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.926057:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018227.926059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018227.926062:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.926065:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.926069:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018227.926073:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018227.926082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335489024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018227.926090:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.926094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018227.926095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018227.926098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335488640:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018227.926099:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.926102:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.926105:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018227.926109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018227.926117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335489152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018227.926125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.926128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018227.926129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018227.926133:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.926135:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.926139:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018227.926143:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018227.926151:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335489280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018227.926159:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018227.926163:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018227.926165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018227.926169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2492 00000100:00080000:0.0:1734018227.926172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018227.926175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018227.926179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018227.926183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018227.926189:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335489408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018227.926211:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59c8700 x1818247335488640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335488640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926253:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335488640:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018227.926260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018227.926278:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018227.926285:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59c9f80 x1818247335488768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335488768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018227.926326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018227.926340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018227.926346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59cb480 x1818247335488896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335488896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335488896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018227.926402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018227.926415:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018227.926421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59cbb80 x1818247335489024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335489024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018227.926452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335489152:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018227.926464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018227.926470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59ca680 x1818247335489152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335489152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335489152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018227.926497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335489280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018227.926509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018227.926515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59c8e00 x1818247335489280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335489280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335489280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018227.926544:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335489408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018227.926556:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018227.926562:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018227/real 1734018227] req@ffff8800b59c8a80 x1818247335489408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018227.926574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335489408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018227.926613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018227.926615:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018227.926619:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335489408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018228.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018228.427961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018229.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018229.429925:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018230.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018230.431976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018231.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018231.433996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018232.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018232.435955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018232.933843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.933852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018232.933856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018232.933867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.933878:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.933886:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018232.933923:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335489536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018232.933959:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.933964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018232.933966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018232.933970:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.933973:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.933977:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018232.933987:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018232.933997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335489664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018232.934008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.934011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018232.934013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018232.934016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.934019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.934022:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018232.934027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018232.934037:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335489792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018232.934046:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.934049:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018232.934050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018232.934054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.934056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.934059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018232.934063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018232.934071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335489920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018232.934080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.934083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018232.934085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018232.934085:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335489536:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018232.934089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.934092:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.934096:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018232.934100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018232.934108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335490048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018232.934116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.934119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018232.934120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018232.934123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.934126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.934129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018232.934133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018232.934140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335490176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018232.934147:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018232.934150:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018232.934152:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018232.934155:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2497 00000100:00080000:0.0:1734018232.934158:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018232.934161:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018232.934165:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018232.934172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335490304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018232.934175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59c9c00 x1818247335489536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335489536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934248:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335489536:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018232.934256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018232.934274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59caa00 x1818247335489664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335489664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018232.934322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018232.934336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59c8a80 x1818247335489792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335489792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934360:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018232.934398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018232.934412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59c8e00 x1818247335489920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335489920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335489920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018232.934451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018232.934464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59ca680 x1818247335490048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934481:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335490048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018232.934538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018232.934555:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934563:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59cbb80 x1818247335490176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335490176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934584:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934587:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018232.934593:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018232.934610:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018232.934616:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018232/real 1734018232] req@ffff8800b59cb480 x1818247335490304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018232.934627:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335490304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018232.934634:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018232.934636:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018232.934640:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018233.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018233.437951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018234.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018234.439985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018235.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018235.441975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018236.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018236.443995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018237.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018237.445957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018237.941832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.941841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018237.941846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.941856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.941866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018237.941873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018237.941910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335490432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018237.941944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.941948:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018237.941950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.941955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.941958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018237.941961:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018237.941970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018237.941981:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335490560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018237.941991:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.941994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018237.941996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.941999:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.942002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018237.942005:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018237.942011:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018237.942019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335490688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018237.942028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.942031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018237.942033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.942036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.942038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018237.942042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018237.942045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018237.942053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335490816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018237.942062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.942065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018237.942067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.942071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.942073:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018237.942074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335490432:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018237.942077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018237.942081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018237.942088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335490944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018237.942097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.942100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018237.942102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.942106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.942108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018237.942112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018237.942116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018237.942123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335491072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018237.942131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018237.942134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018237.942135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018237.942138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2502 00000100:00080000:0.0:1734018237.942141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018237.942144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018237.942148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018237.942157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335491200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018237.942165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942195:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59c9f80 x1818247335490432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335490432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335490432:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018237.942247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335490560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018237.942270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59c8700 x1818247335490560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335490560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942304:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335490560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018237.942310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335490688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018237.942325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59cb100 x1818247335490688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335490688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942349:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335490688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018237.942388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335490816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018237.942402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59cb800 x1818247335490816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335490816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335490816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018237.942482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018237.942498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59cad80 x1818247335490944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335490944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335490944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018237.942535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018237.942547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59c9180 x1818247335491072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335491072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942574:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018237.942584:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018237.942598:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018237.942604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018237/real 1734018237] req@ffff8800b59ca300 x1818247335491200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018237.942614:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335491200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018237.942622:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018237.942625:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018237.942628:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018238.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018238.447979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018239.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018239.449977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018240.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018240.451957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018241.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018241.453975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018242.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018242.455967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018242.949816:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.949825:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018242.949831:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.949844:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.949855:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.949864:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018242.949911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335491328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018242.949956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.949961:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018242.949963:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.949968:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.949972:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.949976:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018242.949987:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018242.949999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335491456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018242.950008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.950012:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018242.950013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.950017:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.950020:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.950024:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018242.950029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018242.950038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335491584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018242.950047:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.950050:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018242.950051:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.950055:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.950058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.950062:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018242.950067:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018242.950068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335491328:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018242.950076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335491712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018242.950085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.950088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018242.950090:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.950094:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.950096:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.950100:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018242.950104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018242.950112:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335491840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018242.950120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.950123:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018242.950124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.950128:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.950130:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.950134:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018242.950138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018242.950147:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335491968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018242.950152:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018242.950155:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018242.950159:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018242.950161:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018242.950165:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2507 00000100:00080000:0.0:1734018242.950168:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018242.950171:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018242.950175:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018242.950178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59c8000 x1818247335491328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018242.950184:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335492096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018242.950200:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335491328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950211:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335491328:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018242.950229:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335491456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018242.950246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018242.950253:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59c9500 x1818247335491456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018242.950263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335491456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950272:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335491456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018242.950284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335491584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018242.950298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018242.950303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59ca300 x1818247335491584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018242.950313:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335491584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335491584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018242.950396:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335491712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018242.950418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018242.950425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59c9180 x1818247335491712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018242.950436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335491712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335491712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018242.950456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018242.950468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018242.950474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59cad80 x1818247335491840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018242.950483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335491840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018242.950501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018242.950513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018242.950519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59cb800 x1818247335491968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018242.950536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335491968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335491968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018242.950554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018242.950566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018242.950572:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018242/real 1734018242] req@ffff8800b59cb100 x1818247335492096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018242.950581:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335492096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018242.950588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018242.950590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018242.950593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018243.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018243.457970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018244.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018244.459987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018245.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018245.461965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018246.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018246.464002:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018247.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018247.465990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018247.957864:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.957872:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018247.957876:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.957888:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.957901:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.957909:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018247.957946:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335492224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018247.957982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.957986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018247.957988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.957993:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.957996:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.957999:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018247.958009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018247.958020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335492352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018247.958031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.958035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018247.958036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.958040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.958042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.958045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018247.958050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018247.958058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335492480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018247.958066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.958069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018247.958071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.958074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.958077:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.958080:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018247.958084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018247.958091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335492608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018247.958099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.958102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018247.958104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.958108:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.958110:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.958114:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018247.958117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018247.958125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335492736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018247.958131:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335492224:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018247.958135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.958138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018247.958140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.958143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.958146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.958149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018247.958153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018247.958160:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335492864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018247.958167:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018247.958170:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018247.958172:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018247.958175:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2512 00000100:00080000:0.0:1734018247.958177:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018247.958180:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018247.958184:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018247.958194:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335492992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018247.958226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59c8700 x1818247335492224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335492224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958340:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335492224:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018247.958353:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335492352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018247.958376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59c9f80 x1818247335492352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335492352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335492352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018247.958416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335492480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018247.958431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59cb480 x1818247335492480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335492480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335492480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018247.958469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335492608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018247.958483:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59cbb80 x1818247335492608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335492608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335492608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018247.958548:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018247.958562:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958568:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59ca680 x1818247335492736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335492736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018247.958599:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018247.958615:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958621:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59c8e00 x1818247335492864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958632:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335492864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958639:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958641:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958644:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018247.958650:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018247.958662:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018247.958668:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018247/real 1734018247] req@ffff8800b59c8a80 x1818247335492992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018247.958678:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335492992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018247.958685:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018247.958687:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018247.958849:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335492992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018248.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018248.467964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018249.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018249.469959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018250.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018250.471943:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018251.213890:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018251.473953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018252.213787:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018252.476031:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018252.965806:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.965822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018252.965829:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.965847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.965863:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.965875:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018252.965927:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335493120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018252.965969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.965973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018252.965975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.965980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.965984:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.965989:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018252.966001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018252.966012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335493248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018252.966023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.966026:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018252.966028:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.966033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.966037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.966041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018252.966046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018252.966056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335493376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018252.966066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.966069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018252.966071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.966075:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.966078:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.966081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018252.966081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335493120:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018252.966085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018252.966093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335493504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018252.966101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.966105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018252.966106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.966110:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.966112:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.966116:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018252.966120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018252.966128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335493632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018252.966142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.966145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018252.966147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.966180:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.966184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.966188:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018252.966193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018252.966202:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335493760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018252.966211:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018252.966215:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018252.966217:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018252.966221:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2517 00000100:00080000:0.0:1734018252.966223:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018252.966226:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018252.966231:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018252.966231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335493248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018252.966239:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335493888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018252.966267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335493376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018252.966284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335493504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018252.966299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335493632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018252.966314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335493760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018252.966330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335493888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018253.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018253.477990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018254.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018254.479991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018255.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018255.481966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018255.971959:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018255.972006:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018255.972027:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018255.972066:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018255.972076:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018255.972084:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018255.972091:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018255.972127:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018255.972137:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018255.972223:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59c8a80 x1818247335493376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335493376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972283:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335493376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018255.972298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59c8e00 x1818247335493504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335493504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335493504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018255.972349:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59ca680 x1818247335493632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335493632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335493632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018255.972382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59cbb80 x1818247335493760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335493760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335493760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018255.972414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59cb480 x1818247335493888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335493888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335493888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018255.972453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59caa00 x1818247335493120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335493120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335493120:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018255.972484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018252/real 1734018255] req@ffff8800b59c9c00 x1818247335493248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018255.972499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335493248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018255.972511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018255.972515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018255.972520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335493248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018256.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018256.483981:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018257.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018257.485962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018257.973817:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.973826:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018257.973831:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.973841:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.973850:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018257.973857:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018257.973893:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335494016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018257.973927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.973932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018257.973934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.973938:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.973942:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018257.973945:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018257.973954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018257.973965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335494144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018257.973975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.973978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018257.973980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.973984:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.973986:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018257.973989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018257.973994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018257.974001:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335494272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018257.974011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.974015:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018257.974016:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.974020:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.974022:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018257.974025:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018257.974029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018257.974037:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335494400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018257.974046:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.974049:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018257.974051:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.974055:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.974057:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018257.974060:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335494016:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018257.974061:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018257.974095:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018257.974105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335494528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018257.974113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.974116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018257.974118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.974122:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.974125:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018257.974129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018257.974133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018257.974143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335494656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018257.974151:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018257.974154:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018257.974155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018257.974159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2522 00000100:00080000:0.0:1734018257.974164:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018257.974167:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018257.974171:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018257.974179:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335494784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018257.974194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59c9f80 x1818247335494016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974248:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335494016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335494016:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018257.974279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335494144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018257.974298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59c8700 x1818247335494144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335494144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335494144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018257.974338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335494272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018257.974353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59cb100 x1818247335494272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335494272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974384:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974387:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974390:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335494272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018257.974424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335494400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018257.974437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59cb800 x1818247335494400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335494400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974462:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335494400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018257.974474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335494528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018257.974486:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59cad80 x1818247335494528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335494528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974509:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335494528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018257.974523:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335494656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018257.974536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59c9180 x1818247335494656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335494656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335494656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018257.974571:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335494784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018257.974583:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018257.974589:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018257/real 1734018257] req@ffff8800b59ca300 x1818247335494784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018257.974600:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335494784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018257.974608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018257.974610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018257.974613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335494784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018258.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018258.487987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018259.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018259.489964:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018260.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018260.491964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018261.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018261.494033:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018262.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018262.496047:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018262.981832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.981841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018262.981845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.981856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.981865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018262.981872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018262.981909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335494912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018262.981944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.981949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018262.981950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.981955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.981959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018262.981962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018262.981972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018262.982021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335495040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018262.982032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.982036:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018262.982037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.982041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.982044:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018262.982048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018262.982053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018262.982063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335495168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018262.982074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.982077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018262.982079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.982083:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.982086:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018262.982089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018262.982094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018262.982097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335494912:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018262.982102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335495296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018262.982110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.982113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018262.982115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.982118:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.982121:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018262.982124:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018262.982128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018262.982135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335495424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018262.982143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.982146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018262.982147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.982151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.982153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018262.982157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018262.982161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018262.982167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335495552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018262.982176:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018262.982179:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018262.982181:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018262.982184:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2527 00000100:00080000:0.0:1734018262.982187:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018262.982188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018262.982190:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018262.982194:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018262.982201:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335495680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018262.982218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59c9500 x1818247335494912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335494912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982262:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335494912:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018262.982270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018262.982289:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018262.982296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59c8000 x1818247335495040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335495040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018262.982352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018262.982367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018262.982373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59ca300 x1818247335495168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335495168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018262.982405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018262.982418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018262.982424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59c9180 x1818247335495296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335495296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018262.982453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335495424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018262.982467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018262.982474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59cad80 x1818247335495424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335495424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335495424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018262.982506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335495552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018262.982519:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018262.982524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59cb800 x1818247335495552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335495552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335495552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018262.982554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335495680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018262.982567:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018262.982573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018262/real 1734018262] req@ffff8800b59cb100 x1818247335495680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018262.982583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335495680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018262.982591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018262.982594:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018262.982597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335495680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018263.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018263.498011:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018264.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018264.500012:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018265.213984:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018265.501993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018266.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018266.504039:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018267.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018267.506033:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018267.989826:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.989835:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018267.989840:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.989850:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.989859:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.989867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018267.989941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335495808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018267.989974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.989979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018267.989981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.989985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.989989:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.989994:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018267.990004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018267.990015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335495936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018267.990025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.990028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018267.990030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.990034:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.990036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.990039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018267.990044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018267.990051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335496064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018267.990060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.990063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018267.990065:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.990068:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.990071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.990074:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018267.990078:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018267.990086:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335496192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018267.990095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.990099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018267.990100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.990104:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.990107:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.990110:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018267.990114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018267.990114:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335495808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018267.990122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335496320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018267.990130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.990133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018267.990134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.990137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.990140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.990143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018267.990147:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018267.990154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335496448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018267.990162:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018267.990165:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018267.990166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018267.990169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2532 00000100:00080000:0.0:1734018267.990171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018267.990174:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018267.990179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018267.990188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335496576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018267.990206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59c8700 x1818247335495808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335495808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990274:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990279:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335495808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018267.990287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018267.990305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990312:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59c9f80 x1818247335495936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990324:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335495936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990338:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335495936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018267.990345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018267.990359:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59c9c00 x1818247335496064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335496064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990388:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018267.990425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018267.990438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59caa00 x1818247335496192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335496192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990462:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990468:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018267.990475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335496320:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018267.990487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59cb480 x1818247335496320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335496320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335496320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018267.990529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335496448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018267.990540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59cbb80 x1818247335496448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335496448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335496448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018267.990575:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335496576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018267.990588:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018267.990593:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018267/real 1734018267] req@ffff8800b59ca680 x1818247335496576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018267.990603:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335496576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018267.990610:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018267.990612:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018267.990615:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335496576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018268.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018268.507984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018269.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018269.509975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018270.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018270.511967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018271.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018271.513954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018272.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018272.516034:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018272.997774:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.997783:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018272.997787:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.997797:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.997807:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.997850:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018272.997886:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335496704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018272.997917:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.997921:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018272.997923:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.997928:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.997931:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.997935:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018272.997945:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018272.997955:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335496832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018272.997966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.997969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018272.997970:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.997974:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.997978:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.997981:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018272.997985:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018272.997995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335496960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018272.998003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.998006:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018272.998008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.998011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.998014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.998017:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018272.998021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018272.998028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335497088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018272.998038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.998041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018272.998042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.998046:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.998048:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.998052:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018272.998056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018272.998056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335496704:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018272.998064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335497216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018272.998078:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.998081:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018272.998083:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.998086:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.998089:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.998092:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018272.998096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018272.998104:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335497344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018272.998111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018272.998114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018272.998116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018272.998119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2537 00000100:00080000:0.0:1734018272.998122:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018272.998125:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018272.998128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018272.998135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335497472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018272.998148:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998177:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59c8e00 x1818247335496704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998199:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335496704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998212:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335496704:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018272.998230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018272.998249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59c8a80 x1818247335496832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335496832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018272.998293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018272.998308:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59ca680 x1818247335496960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335496960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998333:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998336:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335496960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018272.998372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018272.998385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59cbb80 x1818247335497088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335497088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018272.998421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335497216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018272.998433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59cb480 x1818247335497216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335497216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335497216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018272.998469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335497344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018272.998481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59caa00 x1818247335497344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335497344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335497344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018272.998515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335497472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018272.998528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018272.998534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018272/real 1734018272] req@ffff8800b59c9c00 x1818247335497472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018272.998544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335497472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018272.998552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018272.998555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018272.998558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335497472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018273.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018273.517996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018274.213877:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018274.520007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018275.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018275.521984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018276.213953:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018276.524031:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018277.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018277.525989:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018278.005837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.005847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018278.005851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.005863:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.005873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.005881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018278.005918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335497600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018278.005954:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.005958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018278.005960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.005964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.005967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.005971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018278.005981:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018278.005992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335497728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018278.006001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.006004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018278.006006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.006010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.006013:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.006016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018278.006021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018278.006028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335497856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018278.006037:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.006041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018278.006042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.006046:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.006049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.006052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018278.006056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018278.006064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335497984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018278.006074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.006077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018278.006078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.006082:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.006084:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.006088:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018278.006092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018278.006099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335498112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018278.006099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335497600:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018278.006107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.006110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018278.006111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.006114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.006117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.006120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018278.006125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018278.006132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335498240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018278.006140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018278.006143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018278.006144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018278.006148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2542 00000100:00080000:0.0:1734018278.006150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018278.006153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018278.006157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018278.006165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335498368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018278.006196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59c9f80 x1818247335497600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006248:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335497600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335497600:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018278.006278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018278.006298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59c8700 x1818247335497728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335497728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018278.006340:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018278.006355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006361:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59cb100 x1818247335497856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335497856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018278.006420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018278.006433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59cb800 x1818247335497984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335497984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335497984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018278.006469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335498112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018278.006482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59cad80 x1818247335498112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335498112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335498112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018278.006518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335498240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018278.006530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59c9180 x1818247335498240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335498240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006554:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335498240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018278.006564:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335498368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018278.006577:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018278.006583:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018278/real 1734018278] req@ffff8800b59ca300 x1818247335498368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018278.006593:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335498368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018278.006608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018278.006611:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018278.006614:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335498368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018278.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018278.527973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018279.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018279.530035:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018280.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018280.531960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018281.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018281.533961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018282.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018282.535984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018283.013843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.013852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018283.013856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.013867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00080000:0.0:1734018283.013876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.013883:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018283.013918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335498496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018283.013961:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.013965:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018283.013967:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.013971:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00080000:0.0:1734018283.013974:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.013978:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018283.013988:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018283.013999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335498624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018283.014008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.014011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018283.014012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.014016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00080000:0.0:1734018283.014019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.014022:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018283.014026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018283.014036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335498752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018283.014045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.014048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018283.014049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.014053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00080000:0.0:1734018283.014055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.014059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018283.014063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018283.014071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335498880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018283.014080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.014083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018283.014085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.014089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00100000:3.0:1734018283.014089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335498496:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018283.014092:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.014095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018283.014099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018283.014108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335499008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018283.014115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.014118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018283.014120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.014123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00080000:0.0:1734018283.014126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.014129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018283.014133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018283.014141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335499136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018283.014148:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018283.014152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018283.014153:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018283.014157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2547 00000100:00080000:0.0:1734018283.014160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018283.014163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018283.014167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018283.014174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335499264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018283.014180:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59c8000 x1818247335498496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335498496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335498496:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018283.014260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335498624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018283.014279:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014286:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59c9500 x1818247335498624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335498624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335498624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018283.014329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335498752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018283.014344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59ca300 x1818247335498752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335498752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335498752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018283.014406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335498880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018283.014420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59c9180 x1818247335498880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335498880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335498880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018283.014458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018283.014472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59cad80 x1818247335499008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335499008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018283.014508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018283.014520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59cb800 x1818247335499136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335499136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018283.014554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018283.014566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018283.014572:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018283/real 1734018283] req@ffff8800b59cb100 x1818247335499264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018283.014584:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335499264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018283.014592:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018283.014594:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018283.014597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018283.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018283.537997:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018284.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018284.540001:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018285.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018285.541943:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018286.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018286.543985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018287.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018287.546018:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018288.021813:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.021822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018288.021826:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.021837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00080000:0.0:1734018288.021846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.021854:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018288.021890:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335499392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018288.021925:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.021930:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018288.021932:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.021937:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00080000:0.0:1734018288.021940:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.021944:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018288.021954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018288.021965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335499520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018288.021975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.021978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018288.021979:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.021983:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00080000:0.0:1734018288.021986:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.021989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018288.021994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018288.022001:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335499648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018288.022010:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.022013:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018288.022014:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.022018:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00080000:0.0:1734018288.022020:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.022024:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018288.022027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018288.022035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335499776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018288.022050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.022053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018288.022055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.022059:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00100000:3.0:1734018288.022059:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335499392:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018288.022061:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.022064:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018288.022068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018288.022076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335499904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018288.022084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.022087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018288.022088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.022091:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00080000:0.0:1734018288.022094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.022097:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018288.022101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018288.022108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335500032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018288.022116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018288.022119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018288.022120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018288.022123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2552 00000100:00080000:0.0:1734018288.022127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018288.022130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018288.022134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018288.022143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335500160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018288.022152:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022181:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59c8700 x1818247335499392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022204:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335499392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022218:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022222:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022227:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335499392:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018288.022235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335499520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018288.022253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59c9f80 x1818247335499520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335499520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335499520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018288.022301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335499648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018288.022316:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022322:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59c9c00 x1818247335499648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335499648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335499648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018288.022378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335499776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018288.022392:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59caa00 x1818247335499776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022409:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335499776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335499776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018288.022430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018288.022443:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59cb480 x1818247335499904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335499904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022466:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335499904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018288.022478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018288.022489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59cbb80 x1818247335500032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335500032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018288.022524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018288.022540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018288.022546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018288/real 1734018288] req@ffff8800b59ca680 x1818247335500160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018288.022596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335500160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018288.022604:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018288.022606:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018288.022610:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018288.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018288.547979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018289.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018289.549986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018290.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018290.551976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018291.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018291.553982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018292.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018292.555986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018293.029805:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.029814:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018293.029818:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.029828:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.029838:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018293.029846:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018293.029882:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335500288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018293.029916:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.029921:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018293.029923:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.029927:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.029930:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018293.029934:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018293.029943:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018293.029954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335500416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018293.029964:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.029967:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018293.029968:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.029972:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.029974:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018293.029978:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018293.029982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018293.029992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335500544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018293.030000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.030003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018293.030005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.030008:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.030010:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018293.030014:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018293.030018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018293.030026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335500672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018293.030035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.030039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018293.030041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.030045:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.030048:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018293.030050:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335500288:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018293.030051:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018293.030056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018293.030063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335500800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018293.030071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.030074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018293.030075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.030079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.030081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018293.030084:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018293.030088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018293.030095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335500928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018293.030103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018293.030105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018293.030107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018293.030110:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2557 00000100:00080000:0.0:1734018293.030113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018293.030116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018293.030120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018293.030127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335501056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018293.030143:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59c8a80 x1818247335500288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335500288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030212:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030218:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335500288:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018293.030225:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335500416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018293.030248:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030255:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59c8e00 x1818247335500416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335500416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335500416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018293.030287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335500544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018293.030302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59ca680 x1818247335500544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335500544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335500544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018293.030364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335500672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018293.030377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59cbb80 x1818247335500672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335500672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335500672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018293.030421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018293.030434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59cb480 x1818247335500800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335500800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018293.030515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018293.030533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030541:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59caa00 x1818247335500928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335500928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335500928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018293.030574:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018293.030587:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018293.030593:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018293/real 1734018293] req@ffff8800b59c9c00 x1818247335501056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018293.030604:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335501056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018293.030612:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018293.030614:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018293.030617:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018293.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018293.557947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018294.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018294.559957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018295.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018295.561959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018296.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018296.563975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018297.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018297.565942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018298.037843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.037854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018298.037860:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.037874:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.037885:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.037895:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018298.037942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335501184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018298.037979:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.037983:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018298.037985:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.037990:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.037994:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.037997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018298.038009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018298.038021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335501312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018298.038031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.038035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018298.038038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.038042:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.038045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.038049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018298.038054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018298.038063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335501440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018298.038072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.038075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018298.038077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.038080:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.038083:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.038087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038088:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335501184:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018298.038091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018298.038099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335501568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018298.038107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.038111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018298.038112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.038116:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.038118:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.038122:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018298.038126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018298.038134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335501696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018298.038144:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.038147:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018298.038148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.038152:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.038155:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.038158:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018298.038162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018298.038170:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335501824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018298.038170:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018298.038178:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018298.038181:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018298.038182:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018298.038185:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2562 00000100:00080000:0.0:1734018298.038188:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018298.038191:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018298.038195:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018298.038199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59c9f80 x1818247335501184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018298.038205:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335501952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018298.038221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335501184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335501184:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018298.038249:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335501312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018298.038267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59c8700 x1818247335501312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018298.038285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335501312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335501312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018298.038329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335501440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018298.038344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59cb100 x1818247335501440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018298.038361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335501440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335501440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018298.038379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335501568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018298.038431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59cb800 x1818247335501568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018298.038450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335501568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335501568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018298.038470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018298.038482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59cad80 x1818247335501696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018298.038498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335501696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018298.038517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018298.038530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59c9180 x1818247335501824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018298.038553:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335501824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018298.038571:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018298.038584:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018298.038590:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018298/real 1734018298] req@ffff8800b59ca300 x1818247335501952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018298.038599:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335501952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018298.038606:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018298.038608:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018298.038611:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335501952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018298.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018298.567969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018299.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018299.569956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018300.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018300.571941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018301.213767:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018301.573930:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018302.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018302.575956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018303.045848:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.045862:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018303.045867:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.045883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.045894:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018303.045904:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018303.045954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335502080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018303.045993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.045998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018303.045999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.046005:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.046008:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018303.046012:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018303.046024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018303.046036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335502208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018303.046045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.046049:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018303.046051:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.046055:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.046058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018303.046061:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018303.046066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018303.046076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335502336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018303.046087:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.046090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018303.046092:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.046096:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.046098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018303.046101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335502080:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018303.046102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018303.046107:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018303.046116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335502464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018303.046124:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.046127:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018303.046129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.046133:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.046135:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018303.046139:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018303.046143:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018303.046151:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335502592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018303.046159:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.046162:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018303.046163:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.046167:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.046171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018303.046175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018303.046179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018303.046187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335502720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018303.046188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018303.046195:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018303.046198:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018303.046200:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018303.046203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2567 00000100:00080000:0.0:1734018303.046206:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018303.046209:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018303.046213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018303.046221:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335502848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018303.046222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59c9500 x1818247335502080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335502080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335502080:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018303.046284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335502208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018303.046348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018303.046357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59c8000 x1818247335502208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335502208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335502208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018303.046411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335502336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018303.046427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018303.046433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59ca300 x1818247335502336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335502336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335502336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018303.046464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335502464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018303.046477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018303.046483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59c9180 x1818247335502464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335502464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046501:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335502464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018303.046513:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335502592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018303.046526:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018303.046532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59cad80 x1818247335502592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335502592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335502592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018303.046560:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335502720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018303.046576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018303.046582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59cb800 x1818247335502720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046593:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335502720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046600:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335502720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018303.046611:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335502848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018303.046623:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018303.046629:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018303/real 1734018303] req@ffff8800b59cb100 x1818247335502848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018303.046639:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335502848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018303.046646:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018303.046649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018303.046651:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335502848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018303.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018303.577976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018304.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018304.579965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018305.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018305.581938:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018306.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018306.583897:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018307.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018307.585920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018308.053818:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.053827:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018308.053831:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018308.053842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.053851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.053859:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018308.053895:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335502976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018308.053927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.053932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018308.053934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018308.053938:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.053941:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.053945:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018308.053955:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018308.053966:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335503104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018308.053975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.053979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018308.053980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018308.053984:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.053987:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.053990:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018308.053994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018308.054002:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335503232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018308.054010:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.054013:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018308.054014:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018308.054018:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.054021:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.054024:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054027:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335502976:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018308.054029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018308.054036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335503360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018308.054045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.054048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018308.054050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018308.054053:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.054055:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.054059:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018308.054062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018308.054069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335503488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018308.054077:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.054080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018308.054081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018308.054084:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.054087:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.054090:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018308.054094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018308.054094:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018308.054102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335503616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018308.054111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018308.054114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018308.054115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018308.054117:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59c8700 x1818247335502976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018308.054118:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2572 00000100:00080000:0.0:1734018308.054121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018308.054124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018308.054128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018308.054136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335503744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018308.054137:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335502976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054147:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054151:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054155:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335502976:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018308.054163:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335503104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018308.054179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054186:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59c9f80 x1818247335503104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018308.054197:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335503104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054205:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054208:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335503104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018308.054259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335503232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018308.054276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054284:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59c9c00 x1818247335503232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018308.054294:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335503232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335503232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018308.054338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335503360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018308.054350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59caa00 x1818247335503360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018308.054367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335503360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054374:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335503360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018308.054385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335503488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018308.054397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054402:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59cb480 x1818247335503488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018308.054412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335503488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054421:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335503488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018308.054431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335503616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018308.054442:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054448:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59cbb80 x1818247335503616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018308.054458:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335503616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335503616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018308.054475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335503744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018308.054487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018308.054493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018308/real 1734018308] req@ffff8800b59ca680 x1818247335503744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018308.054502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335503744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018308.054510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018308.054512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018308.054515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335503744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018308.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018308.587950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018309.213763:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018309.589976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018310.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018310.591931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018311.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018311.593947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018312.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018312.595966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018313.061830:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018313.061839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018313.061844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018313.061855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.061865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.061872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018313.061908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335503872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018313.061942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018313.061946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018313.061949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018313.061953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.061956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.061960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018313.061969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018313.061980:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335504000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018313.061989:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018313.061992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018313.061994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018313.061998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.062001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.062004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018313.062009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018313.062019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335504128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018313.062027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018313.062030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018313.062031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018313.062035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.062037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.062041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018313.062045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018313.062053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335504256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018313.062062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734018313.062062:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335503872:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018313.062066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018313.062067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018313.062071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.062073:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.062077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018313.062081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018313.062088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335504384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018313.062096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018313.062099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018313.062100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018313.062103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.062106:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.062109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018313.062113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018313.062121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335504512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018313.062158:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018313.062161:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018313.062162:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734018313.062166:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018313.062167:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2577 00000100:00080000:0.0:1734018313.062170:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018313.062174:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018313.062178:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018313.062186:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335504640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018313.062193:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59c8e00 x1818247335503872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335503872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062227:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335503872:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018313.062240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018313.062262:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018313.062268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59c8a80 x1818247335504000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335504000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018313.062299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018313.062312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018313.062318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59ca680 x1818247335504128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335504128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018313.062368:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018313.062380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018313.062386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59cbb80 x1818247335504256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335504256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018313.062414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335504384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018313.062426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018313.062431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59cb480 x1818247335504384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335504384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335504384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018313.062460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335504512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018313.062471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018313.062477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59caa00 x1818247335504512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335504512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335504512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018313.062504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335504640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018313.062515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018313.062521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018313/real 1734018313] req@ffff8800b59c9c00 x1818247335504640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018313.062530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335504640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018313.062538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018313.062539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018313.062542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335504640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018313.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018313.597939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018314.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018314.599951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018315.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018315.601982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018316.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018316.603908:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018317.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018317.605971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018318.069787:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.069796:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018318.069801:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.069812:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.069821:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.069829:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018318.069866:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335504768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018318.069901:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.069905:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018318.069907:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.069911:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.069915:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.069918:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018318.069928:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018318.069940:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335504896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018318.069950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.069954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018318.069955:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.069959:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.069962:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.069965:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018318.069969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018318.069977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335505024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018318.069985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.069988:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018318.069989:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.069993:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.069995:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.069999:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018318.070002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018318.070010:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335505152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018318.070016:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335504768:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018318.070018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.070021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018318.070023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.070027:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.070029:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.070033:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018318.070037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018318.070073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335505280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018318.070081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.070084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018318.070086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.070090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.070092:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.070096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018318.070100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018318.070108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335505408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018318.070116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018318.070119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018318.070120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018318.070124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2582 00000100:00080000:0.0:1734018318.070126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018318.070129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018318.070133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018318.070133:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018318.070142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335505536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018318.070156:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018318.070181:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018318.070196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335505280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018318.070210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335505408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018318.070224:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335505536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018318.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018318.607961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018319.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018319.609958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018320.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018320.611936:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018321.076043:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018321.076090:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018321.076110:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018321.076146:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018321.076156:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018321.076163:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018321.076171:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018321.076178:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018321.076185:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018321.076260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59cb800 x1818247335505152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335505152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018321.076318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59cad80 x1818247335505280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335505280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076340:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335505280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018321.076352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59c9180 x1818247335505408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335505408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335505408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018321.076383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59ca300 x1818247335505536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335505536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335505536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018321.076412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59c9f80 x1818247335504768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335504768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335504768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018321.076450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59c8700 x1818247335504896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335504896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335504896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018321.076479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018318/real 1734018321] req@ffff8800b59cb100 x1818247335505024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018321.076489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335505024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018321.076496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018321.076508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018321.076512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018321.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018321.613949:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018322.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018322.615946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018323.077845:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.077858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018323.077863:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.077878:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.077890:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.077900:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018323.077947:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335505664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018323.078026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.078031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018323.078033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.078038:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.078042:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.078046:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018323.078057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018323.078070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335505792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018323.078081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.078085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018323.078087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.078092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.078095:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.078098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018323.078104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018323.078114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335505920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018323.078123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.078126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018323.078128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.078132:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.078134:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.078138:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018323.078142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018323.078150:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335505664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018323.078152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335506048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018323.078161:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.078164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018323.078166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.078170:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.078173:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.078176:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018323.078181:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018323.078189:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335506176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018323.078198:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.078201:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018323.078203:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.078207:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.078210:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.078213:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018323.078217:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018323.078226:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335506304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018323.078227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018323.078234:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018323.078237:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018323.078238:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018323.078242:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2587 00000100:00080000:0.0:1734018323.078245:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018323.078248:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59c8000 x1818247335505664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018323.078252:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018323.078261:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335506432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018323.078273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335505664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335505664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018323.078303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018323.078320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59c9500 x1818247335505792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018323.078339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335505792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018323.078360:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018323.078374:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078380:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59cb100 x1818247335505920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018323.078391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335505920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335505920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018323.078433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018323.078446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59c8700 x1818247335506048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018323.078464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335506048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018323.078483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335506176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018323.078495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59c9f80 x1818247335506176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018323.078511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335506176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335506176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018323.078530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335506304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018323.078546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078553:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59ca300 x1818247335506304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018323.078563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335506304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335506304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018323.078581:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335506432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018323.078593:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018323.078599:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018323/real 1734018323] req@ffff8800b59c9180 x1818247335506432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018323.078609:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335506432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018323.078616:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018323.078618:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018323.078621:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335506432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018323.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018323.618033:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018324.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018324.619997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018325.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018325.621966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018326.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018326.623995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018327.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018327.625995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018328.085839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.085851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018328.085857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.085908:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.085920:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.085930:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018328.085979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335506560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018328.086017:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.086022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018328.086024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.086030:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.086033:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.086037:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018328.086049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018328.086061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335506688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018328.086071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.086075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018328.086077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.086081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.086084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.086087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018328.086092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018328.086101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335506816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018328.086111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.086114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018328.086116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.086119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.086122:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.086125:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018328.086130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018328.086139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335506944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018328.086139:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335506560:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018328.086148:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.086152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018328.086153:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.086157:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.086159:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.086163:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018328.086168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018328.086176:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335507072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018328.086184:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.086187:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018328.086188:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.086192:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.086196:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.086200:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018328.086204:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018328.086213:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018328.086216:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335507200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018328.086224:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018328.086227:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018328.086228:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018328.086232:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2592 00000100:00080000:0.0:1734018328.086235:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018328.086238:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59cad80 x1818247335506560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018328.086242:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018328.086251:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335507328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018328.086259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335506560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335506560:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018328.086289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018328.086307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59cb800 x1818247335506688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018328.086326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335506688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018328.086347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018328.086361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59c9c00 x1818247335506816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018328.086377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335506816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018328.086426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018328.086439:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59caa00 x1818247335506944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018328.086456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335506944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335506944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018328.086475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018328.086487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59cb480 x1818247335507072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018328.086503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335507072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018328.086524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018328.086536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59cbb80 x1818247335507200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018328.086552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335507200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018328.086570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018328.086581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018328.086587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018328/real 1734018328] req@ffff8800b59ca680 x1818247335507328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018328.086597:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335507328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018328.086604:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018328.086606:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018328.086609:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018328.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018328.627949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018329.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018329.629998:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018330.213903:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018330.631999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018331.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018331.633921:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018332.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018332.636006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018333.093868:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.093877:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018333.093881:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.093892:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.093901:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.093908:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018333.093947:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335507456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018333.093981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.093986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018333.093987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.093992:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.093995:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.093999:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018333.094008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018333.094019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335507584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018333.094028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.094032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018333.094040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.094044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.094046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.094050:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018333.094054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018333.094064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335507712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018333.094072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.094075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018333.094077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.094081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.094083:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.094087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018333.094091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018333.094099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335507840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018333.094100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335507456:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018333.094108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.094111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018333.094112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.094116:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.094118:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.094122:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018333.094126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018333.094133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335507968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018333.094141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.094143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018333.094145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.094149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.094152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.094155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018333.094159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018333.094166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335508096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018333.094167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018333.094175:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018333.094179:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018333.094180:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018333.094183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2597 00000100:00080000:0.0:1734018333.094186:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018333.094189:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59c8a80 x1818247335507456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018333.094193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018333.094200:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335508224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018333.094209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335507456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094221:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094224:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094229:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335507456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018333.094236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335507584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018333.094252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59c8e00 x1818247335507584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018333.094270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335507584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335507584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018333.094290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335507712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018333.094303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59ca680 x1818247335507712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018333.094319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335507712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335507712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018333.094359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335507840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018333.094371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59cbb80 x1818247335507840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018333.094388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335507840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335507840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018333.094406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018333.094417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59cb480 x1818247335507968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018333.094433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335507968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335507968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018333.094451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018333.094462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59caa00 x1818247335508096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018333.094478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335508096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018333.094495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018333.094507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018333.094513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018333/real 1734018333] req@ffff8800b59c9c00 x1818247335508224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018333.094522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335508224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018333.094529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018333.094531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018333.094534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018333.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018333.637966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018334.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018334.639969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018335.213900:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018335.642014:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018336.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018336.643969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018337.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018337.645990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018338.101840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018338.101850:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018338.101854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.101865:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.101875:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.101883:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018338.101920:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335508352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018338.101955:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018338.101959:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018338.101963:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.101967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.101970:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.101974:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018338.101984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018338.101995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335508480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018338.102004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018338.102008:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018338.102010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.102013:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.102016:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.102020:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018338.102024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018338.102032:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335508608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018338.102040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018338.102043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018338.102045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.102049:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.102051:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.102055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018338.102059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018338.102067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335508736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018338.102070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335508352:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018338.102076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018338.102079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018338.102081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.102085:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.102087:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.102090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018338.102094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018338.102101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335508864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018338.102109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018338.102112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018338.102113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.102116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.102119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.102122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018338.102126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018338.102133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335508992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018338.102141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734018338.102141:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018338.102144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018338.102146:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018338.102149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2602 00000100:00080000:0.0:1734018338.102152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018338.102155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018338.102159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018338.102164:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59cb800 x1818247335508352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102183:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335508352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102195:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102203:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335508352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018338.102205:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335509120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018338.102210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335508480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018338.102226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018338.102233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59cad80 x1818247335508480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335508480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335508480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018338.102263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335508608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018338.102276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018338.102282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59c9180 x1818247335508608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335508608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335508608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018338.102334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335508736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018338.102346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018338.102352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59ca300 x1818247335508736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335508736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335508736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018338.102380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018338.102391:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018338.102397:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59c9f80 x1818247335508864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102407:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335508864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018338.102426:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018338.102438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018338.102444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59c8700 x1818247335508992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335508992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102461:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335508992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018338.102471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335509120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018338.102482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018338.102488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018338/real 1734018338] req@ffff8800b59cb100 x1818247335509120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018338.102498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335509120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018338.102505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018338.102507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018338.102510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335509120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018338.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018338.647985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018339.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018339.649972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018340.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018340.651935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018341.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018341.653931:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018342.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018342.655924:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018343.109842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.109851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018343.109855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.109865:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.109874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.109881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018343.109917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335509248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018343.109949:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.109954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018343.109956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.109960:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.109963:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.109967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018343.109976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018343.109987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335509376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018343.109997:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.110000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018343.110002:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.110006:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.110008:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.110012:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018343.110016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018343.110025:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335509504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018343.110033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.110036:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018343.110037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.110041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.110043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.110046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018343.110050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018343.110058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335509632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018343.110058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335509248:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018343.110067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.110070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018343.110071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.110076:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.110078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.110082:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018343.110086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018343.110093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335509760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018343.110100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.110103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018343.110105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.110108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.110111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.110114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018343.110118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018343.110125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335509888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018343.110125:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018343.110133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018343.110135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018343.110137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018343.110140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2607 00000100:00080000:0.0:1734018343.110142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018343.110145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59c9500 x1818247335509248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018343.110150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018343.110157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335510016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018343.110168:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335509248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110178:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110182:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110186:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335509248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018343.110193:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335509376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018343.110210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59c8000 x1818247335509376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018343.110228:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335509376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110236:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110239:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335509376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018343.110248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335509504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018343.110266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59cb100 x1818247335509504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018343.110290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335509504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335509504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018343.110333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335509632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018343.110346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59c8700 x1818247335509632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018343.110362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335509632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335509632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018343.110381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335509760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018343.110392:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59c9f80 x1818247335509760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018343.110408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335509760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110415:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335509760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018343.110428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335509888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018343.110439:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59ca300 x1818247335509888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018343.110454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335509888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110462:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110464:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110467:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335509888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018343.110471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018343.110483:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018343.110489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018343/real 1734018343] req@ffff8800b59c9180 x1818247335510016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018343.110498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335510016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018343.110506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018343.110508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018343.110511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018343.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018343.657916:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018344.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018344.659949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018345.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018345.661944:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018346.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018346.663906:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018347.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018347.666026:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018348.117832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.117841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018348.117845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.117856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.117865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.117872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018348.117909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335510144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018348.117942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.117947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018348.117949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.117953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.117956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.117960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018348.117969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018348.117979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335510272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018348.117989:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.117992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018348.117995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.117998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.118001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.118004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018348.118009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018348.118017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335510400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018348.118025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.118028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018348.118030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.118033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.118035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.118039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018348.118043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018348.118049:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335510144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018348.118050:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335510528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018348.118059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.118062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018348.118063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.118067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.118069:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.118072:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018348.118076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018348.118083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335510656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018348.118092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.118095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018348.118097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.118100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.118102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.118106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018348.118110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018348.118117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335510784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018348.118117:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018348.118125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018348.118128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018348.118129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018348.118132:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2612 00000100:00080000:0.0:1734018348.118135:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018348.118138:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118140:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59cad80 x1818247335510144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018348.118142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018348.118150:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335510912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018348.118160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335510144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118171:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335510144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018348.118186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335510272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018348.118202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59cb800 x1818247335510272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018348.118221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335510272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335510272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018348.118241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335510400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018348.118254:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59c9c00 x1818247335510400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018348.118277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335510400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118290:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335510400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018348.118321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335510528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018348.118334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118340:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59caa00 x1818247335510528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018348.118350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335510528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335510528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018348.118370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018348.118381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59cb480 x1818247335510656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018348.118396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335510656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118408:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018348.118417:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018348.118428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59cbb80 x1818247335510784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018348.118444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335510784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018348.118461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018348.118472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018348.118478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018348/real 1734018348] req@ffff8800b59ca680 x1818247335510912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018348.118488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335510912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018348.118495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018348.118497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018348.118500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335510912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018348.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018348.667981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018349.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018349.669968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018350.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018350.672002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018351.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018351.673982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018352.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018352.675956:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018353.125823:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018353.125833:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018353.125837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.125849:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.125858:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.125865:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018353.125901:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335511040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018353.125935:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018353.125939:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018353.125941:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.125946:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.125949:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.125952:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018353.125962:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018353.125973:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335511168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018353.125982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018353.125985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018353.125987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.125990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.125993:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.125997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018353.126001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018353.126011:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335511296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018353.126020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018353.126023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018353.126025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.126028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.126030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.126034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018353.126038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018353.126045:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335511424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018353.126053:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335511040:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018353.126054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018353.126057:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018353.126058:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.126062:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.126064:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.126067:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018353.126071:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018353.126078:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335511552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018353.126086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018353.126089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018353.126090:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.126094:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.126097:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.126100:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018353.126104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018353.126111:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335511680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018353.126119:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734018353.126120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018353.126122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018353.126123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018353.126126:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2617 00000100:00080000:0.0:1734018353.126129:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018353.126132:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018353.126136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018353.126143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335511808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018353.126143:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59c8e00 x1818247335511040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126164:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335511040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126174:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126177:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126182:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335511040:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018353.126189:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335511168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018353.126206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018353.126213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59c8a80 x1818247335511168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335511168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335511168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018353.126243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335511296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018353.126257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018353.126263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59ca680 x1818247335511296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335511296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335511296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018353.126312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335511424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018353.126325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018353.126331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59cbb80 x1818247335511424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335511424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335511424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018353.126359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335511552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018353.126370:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018353.126376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59cb480 x1818247335511552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335511552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335511552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018353.126406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335511680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018353.126417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018353.126423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59caa00 x1818247335511680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335511680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335511680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018353.126488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335511808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018353.126504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018353.126512:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018353/real 1734018353] req@ffff8800b59c9c00 x1818247335511808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018353.126522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335511808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018353.126530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018353.126532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018353.126535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335511808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018353.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018353.677958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018354.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018354.679928:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018355.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018355.681923:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018356.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018356.683956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018357.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018357.685986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018358.133819:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.133828:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018358.133833:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.133843:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.133852:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.133860:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018358.133897:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335511936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018358.133931:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.133935:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018358.133937:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.133942:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.133945:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.133948:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018358.133958:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018358.133968:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335512064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018358.133978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.133981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018358.133982:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.133986:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.133989:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.133993:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018358.133997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018358.134005:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335512192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018358.134013:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.134016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018358.134017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.134022:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.134024:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.134027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018358.134031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018358.134038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335512320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018358.134044:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335511936:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018358.134047:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.134050:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018358.134052:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.134056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.134058:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.134061:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018358.134065:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018358.134072:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335512448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018358.134080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.134082:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018358.134084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.134087:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.134090:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.134093:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018358.134097:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018358.134104:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335512576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018358.134112:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018358.134113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018358.134116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018358.134118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018358.134121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2622 00000100:00080000:0.0:1734018358.134124:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018358.134127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018358.134131:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018358.134135:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59cb800 x1818247335511936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018358.134140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335512704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018358.134156:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335511936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134167:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134170:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134175:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335511936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018358.134182:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018358.134198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018358.134205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59cad80 x1818247335512064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018358.134216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335512064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134227:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134230:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018358.134236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018358.134254:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018358.134260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59c9180 x1818247335512192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018358.134271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335512192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134279:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018358.134311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018358.134323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018358.134330:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59ca300 x1818247335512320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018358.134339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335512320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134349:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134352:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018358.134357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335512448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018358.134420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018358.134428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59c9f80 x1818247335512448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018358.134438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335512448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335512448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018358.134459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335512576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018358.134471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018358.134477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59c8700 x1818247335512576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018358.134486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335512576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335512576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018358.134505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335512704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018358.134517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018358.134523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018358/real 1734018358] req@ffff8800b59cb100 x1818247335512704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018358.134533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335512704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018358.134540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018358.134542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018358.134545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335512704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018358.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018358.687950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018359.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018359.689943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018360.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018360.691946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018361.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018361.693960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018362.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018362.695936:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018363.141822:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.141836:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018363.141841:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.141856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.141868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.141878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018363.141926:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335512832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018363.141966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.141971:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018363.141974:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.141979:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.141983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.141987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018363.141999:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018363.142010:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335512960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018363.142020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.142024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018363.142026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.142030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.142033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.142036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018363.142041:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018363.142052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335513088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018363.142061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.142064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018363.142066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.142069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.142072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.142076:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018363.142080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018363.142089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335513216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018363.142098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335512832:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018363.142100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.142104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018363.142106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.142110:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.142112:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.142116:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018363.142120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018363.142128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335513344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018363.142137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.142140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018363.142141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.142145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.142148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.142151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018363.142155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018363.142163:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335513472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018363.142170:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018363.142173:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1734018363.142174:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018363.142175:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018363.142178:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2627 00000100:00080000:0.0:1734018363.142184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018363.142188:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018363.142192:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018363.142200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59c8000 x1818247335512832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018363.142201:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335513600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018363.142221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335512832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142238:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335512832:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018363.142251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018363.142269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018363.142316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59c9500 x1818247335512960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018363.142328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335512960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142343:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335512960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018363.142350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018363.142367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018363.142374:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59cb100 x1818247335513088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018363.142385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335513088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018363.142427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018363.142440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018363.142447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59c8700 x1818247335513216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018363.142457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335513216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018363.142477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335513344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018363.142490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018363.142496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59c9f80 x1818247335513344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018363.142506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335513344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335513344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018363.142527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335513472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018363.142538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018363.142544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59ca300 x1818247335513472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018363.142554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335513472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335513472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018363.142572:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335513600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018363.142584:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018363.142590:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018363/real 1734018363] req@ffff8800b59c9180 x1818247335513600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018363.142600:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335513600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018363.142607:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018363.142609:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018363.142612:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335513600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018363.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018363.697930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018364.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018364.699958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018365.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018365.701953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018366.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018366.703931:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018367.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018367.705955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018368.149840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018368.149849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018368.149854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.149865:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.149874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.149882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018368.149919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335513728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018368.149951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018368.149956:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018368.149958:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.149963:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.149966:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.149970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018368.149979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018368.149991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335513856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018368.150000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018368.150004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018368.150005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.150009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.150012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.150015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018368.150019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018368.150028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335513984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018368.150036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018368.150039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018368.150040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.150044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.150046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.150049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018368.150053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018368.150061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335514112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018368.150067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335513728:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018368.150069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018368.150072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018368.150074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.150078:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.150081:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.150084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018368.150088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018368.150095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335514240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018368.150103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018368.150106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018368.150107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.150111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.150113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.150117:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018368.150120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018368.150127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335514368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018368.150135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734018368.150135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018368.150138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018368.150139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018368.150143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2632 00000100:00080000:0.0:1734018368.150145:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018368.150148:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018368.150152:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018368.150158:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59cad80 x1818247335513728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018368.150162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335514496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018368.150178:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335513728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335513728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018368.150243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018368.150265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018368.150274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59cb800 x1818247335513856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018368.150285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335513856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150296:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018368.150305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018368.150318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018368.150324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59c9c00 x1818247335513984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018368.150335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335513984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335513984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018368.150375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335514112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018368.150387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018368.150393:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59caa00 x1818247335514112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018368.150403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335514112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335514112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018368.150421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335514240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018368.150433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018368.150439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59cb480 x1818247335514240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018368.150448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335514240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335514240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018368.150467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335514368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018368.150479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018368.150485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59cbb80 x1818247335514368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018368.150494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335514368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335514368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018368.150512:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335514496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018368.150523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018368.150529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018368/real 1734018368] req@ffff8800b59ca680 x1818247335514496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018368.150539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335514496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018368.150546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018368.150548:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018368.150551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335514496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018368.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018368.707973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018369.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018369.709964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018370.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018370.711929:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018371.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018371.713966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018372.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018372.715896:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018373.157858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.157868:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018373.157873:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.157883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.157893:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.157901:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018373.157938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335514624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018373.157971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.157976:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018373.157978:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.157982:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.157985:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.157989:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018373.157998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018373.158009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335514752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018373.158019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.158022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018373.158025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.158028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.158031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.158034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018373.158039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018373.158049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335514880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018373.158057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.158060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018373.158062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.158065:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.158068:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.158071:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018373.158075:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018373.158083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335515008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018373.158088:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335514624:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018373.158092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.158096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018373.158097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.158101:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.158137:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.158142:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018373.158146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018373.158155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335515136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018373.158165:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.158168:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018373.158169:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.158173:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.158175:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.158178:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018373.158182:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018373.158190:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335515264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018373.158196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018373.158198:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018373.158201:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018373.158202:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018373.158206:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2637 00000100:00080000:0.0:1734018373.158208:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018373.158211:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018373.158215:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018373.158219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59c8a80 x1818247335514624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018373.158223:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335515392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018373.158239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335514624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335514624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018373.158265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335514752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018373.158282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018373.158289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59c8e00 x1818247335514752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018373.158300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335514752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335514752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018373.158320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335514880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018373.158334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018373.158340:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59ca680 x1818247335514880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018373.158351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335514880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335514880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018373.158389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018373.158402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018373.158409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59cbb80 x1818247335515008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018373.158419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335515008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018373.158437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335515136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018373.158448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018373.158454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59cb480 x1818247335515136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018373.158464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335515136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335515136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018373.158483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335515264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018373.158495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018373.158501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59caa00 x1818247335515264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018373.158510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335515264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335515264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018373.158527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335515392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018373.158539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018373.158545:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018373/real 1734018373] req@ffff8800b59c9c00 x1818247335515392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018373.158554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335515392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018373.158561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018373.158563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018373.158566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335515392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018373.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018373.717883:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018374.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018374.719880:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018375.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018375.721906:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018376.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018376.723874:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018377.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018377.725906:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018378.165857:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018378.165871:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018378.165877:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018378.165891:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.165904:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018378.165914:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018378.165964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335515520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018378.166005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018378.166009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018378.166011:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018378.166017:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.166052:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018378.166057:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018378.166069:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018378.166082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335515648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018378.166094:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018378.166097:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018378.166099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018378.166103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.166106:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018378.166109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018378.166114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018378.166122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335515776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018378.166134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018378.166137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018378.166139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018378.166141:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335515520:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018378.166143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.166145:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018378.166149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018378.166153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018378.166162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335515904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018378.166171:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018378.166174:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018378.166175:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018378.166179:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.166181:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018378.166186:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018378.166190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018378.166198:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335516032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018378.166207:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018378.166210:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018378.166211:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018378.166215:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.166219:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018378.166219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018378.166223:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018378.166227:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018378.166235:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335516160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018378.166243:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734018378.166245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59cb800 x1818247335515520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018378.166246:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018378.166248:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018378.166251:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2642 00000100:00080000:0.0:1734018378.166254:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018378.166258:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018378.166262:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018378.166266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335515520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018378.166272:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335516288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734018378.166278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335515520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018378.166295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018378.166313:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018378.166320:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59cad80 x1818247335515648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018378.166331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335515648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018378.166339:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018378.166370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018378.166384:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018378.166391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59c9180 x1818247335515776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018378.166402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335515776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018378.166410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018378.166422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018378.166434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018378.166439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59ca300 x1818247335515904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018378.166449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335515904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018378.166457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335515904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018378.166468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018378.166481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018378.166487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59c9f80 x1818247335516032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018378.166497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335516032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018378.166504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018378.166517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018378.166530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018378.166535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59c8700 x1818247335516160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018378.166545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335516160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018378.166553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018378.166564:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018378.166576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018378.166582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018378/real 1734018378] req@ffff8800b59cb100 x1818247335516288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018378.166592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335516288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018378.166599:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018378.166601:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018378.166604:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018378.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018378.727913:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018379.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018379.729959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018380.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018380.731946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018381.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018381.734003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018382.213926:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018382.736018:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018383.173839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.173849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018383.173853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.173864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.173873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.173882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018383.173919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335516416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018383.173989:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.173993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018383.173995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.174001:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.174005:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.174009:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018383.174019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018383.174034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335516544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018383.174044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.174047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018383.174049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.174052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.174054:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.174058:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018383.174062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018383.174072:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335516672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018383.174081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.174083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018383.174085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.174088:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.174091:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.174096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018383.174100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018383.174101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335516416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018383.174108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335516800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018383.174117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.174120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018383.174122:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.174126:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.174128:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.174131:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018383.174135:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018383.174142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335516928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018383.174150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.174153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018383.174155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.174158:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.174161:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.174164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018383.174168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018383.174175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335517056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018383.174182:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018383.174186:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018383.174188:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018383.174191:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2647 00000100:00080000:0.0:1734018383.174193:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018383.174197:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018383.174198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335516544:192.168.201.104@tcp:38:kworker.0 00000008:00100000:0.0:1734018383.174201:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018383.174208:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335517184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018383.174219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335516672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018383.174244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335516800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018383.174259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018383.174275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018383.174289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018383.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018383.738004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018384.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018384.740010:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018385.213796:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018385.741970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018386.180045:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018386.180093:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018386.180115:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018386.180151:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018386.180161:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018386.180169:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018386.180177:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018386.180185:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018386.180191:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018386.180275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59c8700 x1818247335516800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335516800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180343:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335516800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018386.180354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59c9f80 x1818247335516928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335516928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335516928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018386.180418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59ca300 x1818247335517056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335517056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018386.180466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59c9180 x1818247335517184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335517184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018386.180510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59c9500 x1818247335516416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335516416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335516416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018386.180569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59c8000 x1818247335516544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335516544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180610:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335516544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018386.180620:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018383/real 1734018386] req@ffff8800b59cb100 x1818247335516672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018386.180637:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335516672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018386.180648:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018386.180652:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018386.180656:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335516672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018386.213793:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018386.743946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018387.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018387.746024:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018388.181811:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018388.181824:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018388.181829:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.181843:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.181892:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.181903:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018388.181952:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335517312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018388.181990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018388.181995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018388.181997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.182002:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.182006:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.182009:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018388.182021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018388.182033:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335517440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018388.182045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018388.182048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018388.182050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.182054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.182056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.182060:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018388.182064:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018388.182073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335517568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018388.182082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018388.182085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018388.182087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.182090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.182093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.182096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018388.182100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018388.182110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335517696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018388.182117:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335517312:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018388.182119:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018388.182122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018388.182124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.182128:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.182132:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.182136:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018388.182140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018388.182148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335517824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018388.182156:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018388.182159:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018388.182161:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.182164:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.182167:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.182170:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018388.182174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018388.182184:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335517952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018388.182192:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734018388.182193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018388.182195:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018388.182196:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018388.182200:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2652 00000100:00080000:0.0:1734018388.182203:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018388.182207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018388.182211:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018388.182218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59cad80 x1818247335517312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018388.182219:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335518080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018388.182239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335517312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182251:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182260:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335517312:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018388.182268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335517440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018388.182286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018388.182293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59cb800 x1818247335517440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018388.182304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335517440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335517440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018388.182325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335517568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018388.182338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018388.182345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59c9c00 x1818247335517568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018388.182355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335517568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335517568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018388.182404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335517696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018388.182417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018388.182423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59caa00 x1818247335517696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018388.182434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335517696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335517696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018388.182454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018388.182466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018388.182472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59cb480 x1818247335517824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018388.182482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335517824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018388.182501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018388.182513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018388.182519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59cbb80 x1818247335517952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018388.182528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335517952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335517952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018388.182547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018388.182558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018388.182564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018388/real 1734018388] req@ffff8800b59ca680 x1818247335518080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018388.182574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335518080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018388.182581:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018388.182583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018388.182586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018388.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018388.747985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018389.213959:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018389.749973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018390.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018390.752000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018391.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018391.753977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018392.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018392.755991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018393.189878:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.189889:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018393.189895:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018393.189909:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.189921:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018393.189930:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018393.189978:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335518208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018393.190016:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.190021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018393.190023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018393.190028:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.190031:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018393.190035:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018393.190047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018393.190058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335518336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018393.190067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.190071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018393.190073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018393.190078:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.190081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018393.190085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018393.190090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018393.190101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335518464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018393.190110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.190113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018393.190114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018393.190118:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.190121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018393.190124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018393.190129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018393.190138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335518592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018393.190148:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.190151:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018393.190153:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018393.190155:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335518208:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018393.190157:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.190160:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018393.190163:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018393.190168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018393.190176:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335518720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018393.190186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.190189:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018393.190191:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018393.190194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.190197:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018393.190200:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018393.190204:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018393.190212:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335518848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018393.190220:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018393.190223:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018393.190225:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018393.190228:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2657 00000100:00080000:0.0:1734018393.190231:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018393.190231:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018393.190235:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018393.190239:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018393.190247:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335518976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018393.190257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59c8e00 x1818247335518208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335518208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190290:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190293:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190298:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335518208:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018393.190306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335518336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018393.190324:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018393.190332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59c8a80 x1818247335518336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335518336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335518336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018393.190363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335518464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018393.190378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018393.190384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59ca680 x1818247335518464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335518464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190411:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190414:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335518464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018393.190444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335518592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018393.190457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018393.190463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59cbb80 x1818247335518592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190473:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335518592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335518592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018393.190493:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018393.190505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018393.190511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59cb480 x1818247335518720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335518720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018393.190540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018393.190551:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018393.190557:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59caa00 x1818247335518848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335518848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018393.190585:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018393.190597:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018393.190603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018393/real 1734018393] req@ffff8800b59c9c00 x1818247335518976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018393.190613:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335518976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018393.190620:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018393.190622:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018393.190625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335518976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018393.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018393.758004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018394.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018394.759971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018395.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018395.761954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018396.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018396.763984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018397.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018397.766022:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018398.197831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.197844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018398.197849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.197864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.197877:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.197887:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018398.197936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335519104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018398.197977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.197982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018398.197984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.197989:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.197992:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.197996:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018398.198008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018398.198020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335519232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018398.198029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.198033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018398.198035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.198039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.198042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.198046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018398.198050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018398.198058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335519360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018398.198069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.198072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018398.198074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.198077:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.198080:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.198083:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018398.198088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018398.198097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335519488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018398.198105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.198108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018398.198110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.198114:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.198116:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.198120:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018398.198124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018398.198133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335519616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018398.198134:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335519104:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018398.198142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.198145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018398.198147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.198151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.198154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.198158:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018398.198162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018398.198171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335519744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018398.198179:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018398.198182:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018398.198183:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018398.198187:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2662 00000100:00080000:0.0:1734018398.198191:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018398.198194:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018398.198199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018398.198209:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335519872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018398.198215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198241:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59cb800 x1818247335519104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335519104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335519104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018398.198291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335519232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018398.198309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59cad80 x1818247335519232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335519232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335519232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018398.198348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335519360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018398.198361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59cb100 x1818247335519360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335519360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335519360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018398.198404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335519488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018398.198416:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198422:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59c8000 x1818247335519488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335519488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335519488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018398.198477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335519616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018398.198490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198497:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59c9500 x1818247335519616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335519616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335519616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018398.198527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335519744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018398.198539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198545:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59c9180 x1818247335519744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335519744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198568:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335519744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018398.198573:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335519872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018398.198585:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018398.198591:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018398/real 1734018398] req@ffff8800b59ca300 x1818247335519872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018398.198600:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335519872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018398.198608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018398.198610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018398.198614:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335519872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018398.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018398.767964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018399.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018399.769984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018400.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018400.771956:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018401.213925:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018401.773922:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018402.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018402.775946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018403.205832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.205843:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018403.205847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.205858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.205868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.205875:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018403.205912:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335520000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018403.205965:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.205968:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018403.205969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.205972:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.205974:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.205977:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018403.205984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018403.205991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335520128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018403.205997:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.205999:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018403.206000:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.206003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.206004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.206007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018403.206026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018403.206037:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335520256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018403.206045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.206048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018403.206050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.206053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.206056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.206060:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018403.206064:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018403.206073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335520384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018403.206081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.206084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018403.206085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.206089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.206091:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.206095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018403.206098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018403.206105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335520512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018403.206113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.206116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018403.206117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.206121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.206123:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.206126:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018403.206130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018403.206136:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335520000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018403.206137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335520640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018403.206146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018403.206149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018403.206151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018403.206154:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2667 00000100:00080000:0.0:1734018403.206157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018403.206160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018403.206164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018403.206171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335520768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018403.206206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59c9f80 x1818247335520000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335520000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335520000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018403.206277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335520128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018403.206293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59c8700 x1818247335520128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335520128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206319:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335520128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018403.206331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335520256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018403.206344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59ca300 x1818247335520256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335520256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206381:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335520256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018403.206386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335520384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018403.206398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59c9180 x1818247335520384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335520384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206421:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335520384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018403.206432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335520512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018403.206444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59c9500 x1818247335520512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335520512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335520512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018403.206501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335520640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018403.206514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59c8000 x1818247335520640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335520640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335520640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018403.206549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335520768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018403.206560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018403.206566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018403/real 1734018403] req@ffff8800b59cb100 x1818247335520768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018403.206576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335520768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018403.206583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018403.206585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018403.206588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335520768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018403.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018403.777965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018404.213938:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018404.779945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018405.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018405.781947:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018406.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018406.783952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018407.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018407.785970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018408.213852:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.213862:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018408.213867:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018408.213877:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2672 00000100:00080000:0.0:1734018408.213885:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.213893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018408.213927:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335520896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:2.0:1734018408.213938:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018408.213957:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.213963:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018408.213966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018408.213973:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2672 00000100:00080000:0.0:1734018408.213978:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.213984:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018408.213998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018408.214016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335521024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018408.214024:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335520896:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018408.214032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.214035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018408.214038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018408.214043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2672 00000100:00080000:0.0:1734018408.214047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.214052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018408.214058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018408.214070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335521152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018408.214082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.214086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018408.214089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018408.214094:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2672 00000800:00000100:3.0:1734018408.214097:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018408.214098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.214103:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018408.214109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018408.214120:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335521280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018408.214122:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59cad80 x1818247335520896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018408.214134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.214140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018408.214142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018408.214145:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335520896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018408.214147:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2672 00000100:00080000:0.0:1734018408.214151:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.214156:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734018408.214160:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1734018408.214162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1734018408.214164:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214170:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335520896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018408.214174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335521408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018408.214186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.214190:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018408.214192:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018408.214198:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2672 00000100:00100000:3.0:1734018408.214198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521024:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734018408.214202:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.214207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018408.214213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018408.214224:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018408.214225:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335521536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018408.214235:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59cb800 x1818247335521024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018408.214239:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018408.214244:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018408.214246:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018408.214251:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2672 00000100:00100000:3.0:1734018408.214253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335521024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018408.214255:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018408.214260:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734018408.214265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1734018408.214267:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734018408.214269:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018408.214279:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335521664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018408.214284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018408.214306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018408.214317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59c9c00 x1818247335521152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018408.214335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335521152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018408.214357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018408.214361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018408.214376:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018408.214400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018408.214409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59caa00 x1818247335521280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018408.214420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335521280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018408.214428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018408.214430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018408.214439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335521408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018408.214451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018408.214457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59cb480 x1818247335521408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018408.214467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335521408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018408.214474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018408.214477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335521408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018408.214500:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335521536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018408.214549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018408.214557:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59cbb80 x1818247335521536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018408.214568:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335521536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018408.214576:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018408.214578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335521536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018408.214587:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335521664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018408.214600:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018408.214607:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018408/real 1734018408] req@ffff8800b59ca680 x1818247335521664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018408.214617:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335521664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018408.214624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018408.214626:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018408.214629:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335521664:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018408.787913:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018409.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018409.789965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018410.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018410.791972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018411.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018411.793951:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018412.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018412.795968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018413.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018413.221802:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.221810:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018413.221815:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.221824:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.221834:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018413.221840:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018413.221877:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335521792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018413.221909:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.221913:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018413.221915:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.221919:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.221923:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018413.221926:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018413.221936:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018413.221946:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335521920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018413.221956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.221959:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018413.221960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.221965:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.221967:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018413.221971:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018413.221975:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018413.221985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335522048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018413.221993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.221996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018413.221998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.222001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.222003:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018413.222007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018413.222010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018413.222015:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335521792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018413.222018:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335522176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018413.222027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.222030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018413.222032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.222035:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.222038:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018413.222041:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018413.222045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018413.222052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335522304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018413.222061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.222064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018413.222066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.222069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.222071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018413.222074:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018413.222078:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018413.222081:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018413.222085:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335522432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018413.222093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018413.222096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018413.222097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018413.222100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2677 00000100:00080000:0.0:1734018413.222103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018413.222104:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59c8a80 x1818247335521792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018413.222106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018413.222110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018413.222117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335522560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018413.222123:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335521792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222134:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222137:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222142:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335521792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018413.222149:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018413.222165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018413.222172:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59c8e00 x1818247335521920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018413.222183:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335521920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222190:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222193:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222196:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335521920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018413.222202:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018413.222216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018413.222222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59ca680 x1818247335522048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018413.222232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335522048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222249:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018413.222278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018413.222290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018413.222296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59cbb80 x1818247335522176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018413.222306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335522176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018413.222324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335522304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018413.222335:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018413.222341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59cb480 x1818247335522304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018413.222351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335522304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335522304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018413.222369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335522432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018413.222381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018413.222387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59caa00 x1818247335522432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018413.222397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335522432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335522432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018413.222414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335522560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018413.222472:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018413.222480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018413/real 1734018413] req@ffff8800b59c9c00 x1818247335522560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018413.222490:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335522560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018413.222497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018413.222499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018413.222502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335522560:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018413.797949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018414.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018414.799964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018415.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018415.801983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018416.213939:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018416.803965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018417.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018417.805961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018418.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018418.229814:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.229822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018418.229827:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.229837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.229846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.229854:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018418.229890:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335522688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018418.229923:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.229927:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018418.229929:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.229934:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.229937:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.229940:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018418.229951:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018418.229961:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335522816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018418.229971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.229974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018418.229976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.229980:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.229982:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.229985:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018418.229990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018418.229997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335522944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018418.230005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.230009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018418.230011:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.230014:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.230017:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.230020:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018418.230024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018418.230032:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335523072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018418.230034:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335522688:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018418.230040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.230043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018418.230045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.230048:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.230051:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.230054:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018418.230058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018418.230065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335523200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018418.230072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.230075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018418.230077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.230080:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.230082:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.230086:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018418.230091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018418.230098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335523328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018418.230102:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018418.230107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018418.230110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018418.230111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018418.230114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2682 00000100:00080000:0.0:1734018418.230117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018418.230120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018418.230124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018418.230125:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59cb800 x1818247335522688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018418.230131:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335523456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018418.230144:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335522688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230155:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230158:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230163:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335522688:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018418.230170:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018418.230187:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018418.230194:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59cad80 x1818247335522816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018418.230205:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335522816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230213:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230215:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230219:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018418.230225:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018418.230238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018418.230244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59cb100 x1818247335522944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018418.230254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335522944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230268:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335522944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018418.230300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018418.230312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018418.230318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59c8000 x1818247335523072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018418.230329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335523072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230374:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018418.230392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335523200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018418.230413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018418.230421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59c9500 x1818247335523200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018418.230433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335523200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230446:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335523200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018418.230452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335523328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018418.230464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018418.230470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59c9180 x1818247335523328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018418.230480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335523328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335523328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018418.230497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335523456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018418.230509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018418.230515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018418/real 1734018418] req@ffff8800b59ca300 x1818247335523456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018418.230525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335523456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018418.230531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018418.230534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018418.230537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335523456:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018418.807906:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018419.213913:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018419.809959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018420.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018420.811935:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018421.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018421.813987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018422.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018422.815998:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018423.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018423.237810:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.237819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018423.237825:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018423.237838:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2687 00000100:00080000:0.0:1734018423.237850:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.237859:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018423.237907:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335523584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018423.237953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.237958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018423.237960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018423.237965:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2687 00000100:00080000:0.0:1734018423.237968:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.237972:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018423.237983:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018423.237994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335523712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018423.238004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.238007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018423.238009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018423.238013:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2687 00000100:00080000:0.0:1734018423.238015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.238019:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018423.238024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018423.238035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335523840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018423.238043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.238046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018423.238048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018423.238051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2687 00000100:00080000:0.0:1734018423.238054:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.238058:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018423.238063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018423.238071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335523968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018423.238081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.238084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018423.238086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018423.238086:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335523584:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018423.238090:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2687 00000100:00080000:0.0:1734018423.238093:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.238096:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018423.238101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018423.238109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335524096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018423.238117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.238120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018423.238121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018423.238125:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2687 00000100:00080000:0.0:1734018423.238127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.238131:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018423.238136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018423.238144:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335524224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018423.238152:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018423.238156:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018423.238157:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018423.238161:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2687 00000800:00000100:3.0:1734018423.238163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018423.238165:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018423.238168:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018423.238172:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018423.238181:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335524352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018423.238190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59c8700 x1818247335523584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238210:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335523584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238222:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238231:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335523584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018423.238239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018423.238297:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018423.238306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59c9f80 x1818247335523712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335523712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018423.238339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018423.238355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018423.238362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59ca300 x1818247335523840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335523840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018423.238421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018423.238434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018423.238440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59c9180 x1818247335523968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335523968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335523968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018423.238469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018423.238481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018423.238487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59c9500 x1818247335524096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335524096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018423.238517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018423.238528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018423.238535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59c8000 x1818247335524224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335524224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018423.238563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018423.238574:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018423.238580:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018423/real 1734018423] req@ffff8800b59cb100 x1818247335524352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018423.238590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335524352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018423.238597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018423.238599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018423.238602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524352:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018423.817975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018424.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018424.819956:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018425.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018425.821954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018426.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018426.823934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018427.213792:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018427.825911:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018428.213793:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018428.245848:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.245856:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018428.245861:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.245870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00080000:0.0:1734018428.245880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.245888:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018428.245923:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335524480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018428.245956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.245960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018428.245963:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.245967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00080000:0.0:1734018428.245971:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.245975:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018428.245984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018428.245995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335524608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018428.246005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.246008:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018428.246009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.246013:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00080000:0.0:1734018428.246015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.246019:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018428.246023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018428.246031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335524736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018428.246039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.246042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018428.246044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.246047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00080000:0.0:1734018428.246049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.246053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018428.246056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018428.246062:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335524480:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018428.246064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335524864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018428.246080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.246083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018428.246084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.246088:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00080000:0.0:1734018428.246090:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.246094:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018428.246097:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018428.246105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335524992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018428.246112:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.246115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018428.246117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.246120:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00080000:0.0:1734018428.246122:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.246126:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018428.246130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018428.246131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018428.246137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335525120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018428.246145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018428.246148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018428.246149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018428.246153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2692 00000100:00100000:3.0:1734018428.246154:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59cad80 x1818247335524480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018428.246157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018428.246160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018428.246164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018428.246204:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335525248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018428.246209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335524480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335524480:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018428.246235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335524608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018428.246255:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018428.246262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59cb800 x1818247335524608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018428.246273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335524608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335524608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018428.246293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335524736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018428.246306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018428.246312:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59c9c00 x1818247335524736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018428.246322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335524736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335524736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018428.246367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335524864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018428.246380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018428.246386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59caa00 x1818247335524864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018428.246396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335524864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335524864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018428.246414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018428.246426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018428.246432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59cb480 x1818247335524992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018428.246441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335524992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246449:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335524992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018428.246460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335525120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018428.246471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018428.246477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59cbb80 x1818247335525120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018428.246487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335525120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335525120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018428.246504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335525248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018428.246516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018428.246522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018428/real 1734018428] req@ffff8800b59ca680 x1818247335525248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018428.246531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335525248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018428.246538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018428.246540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018428.246543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335525248:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018428.827960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018429.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018429.829926:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018430.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018430.831967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018431.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018431.833966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018432.213771:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018432.836003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018433.213765:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018433.253815:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018433.253823:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018433.253827:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.253837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.253847:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.253855:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018433.253894:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335525376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018433.253928:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018433.253932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018433.253934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.253939:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.253942:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.253945:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018433.253955:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018433.253965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335525504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018433.253975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018433.253978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018433.253980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.253984:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.253986:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.253990:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018433.254002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018433.254012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335525632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018433.254020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018433.254023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018433.254025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.254028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.254030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.254033:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018433.254038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018433.254043:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335525376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018433.254046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335525760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018433.254055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018433.254068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018433.254070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.254074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.254076:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.254080:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018433.254130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018433.254138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335525888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018433.254145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018433.254148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018433.254149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.254152:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.254155:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.254158:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018433.254161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018433.254161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018433.254167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335526016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018433.254174:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734018433.254176:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59c8e00 x1818247335525376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018433.254177:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018433.254178:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018433.254181:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2697 00000100:00080000:0.0:1734018433.254184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018433.254186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335525376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018433.254190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734018433.254193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734018433.254196:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335526144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018433.254197:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335525376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018433.254202:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335525504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018433.254211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59c8a80 x1818247335525504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018433.254221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335525504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018433.254224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018433.254227:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335525504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018433.254230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335525632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018433.254237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59ca680 x1818247335525632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018433.254245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335525632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018433.254253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018433.254256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335525632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018433.254271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335525760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018433.254277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59cbb80 x1818247335525760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018433.254285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335525760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018433.254289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018433.254292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335525760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018433.254295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335525888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018433.254301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59cb480 x1818247335525888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018433.254309:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335525888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018433.254312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018433.254315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335525888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018433.254318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018433.254324:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59caa00 x1818247335526016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018433.254349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335526016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018433.254357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018433.254362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018433.254367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018433.254381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018433.254387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018433/real 1734018433] req@ffff8800b59c9c00 x1818247335526144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018433.254397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335526144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018433.254404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018433.254406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018433.254409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526144:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018433.837968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018434.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018434.839955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018435.213771:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018435.841921:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018436.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018436.843945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018437.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018437.845957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018438.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018438.261836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.261844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018438.261848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.261858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.261867:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.261874:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018438.261912:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335526272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018438.261945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.261949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018438.261951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.261955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.261959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.261962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018438.261972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018438.261983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335526400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018438.261992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.262034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018438.262050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.262055:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.262058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.262062:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018438.262068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018438.262078:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335526528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018438.262080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335526272:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018438.262088:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.262091:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018438.262093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.262097:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.262099:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.262102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018438.262106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018438.262114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335526656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018438.262123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.262126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018438.262128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.262131:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.262134:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.262137:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018438.262141:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018438.262148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335526784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018438.262151:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018438.262157:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.262160:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018438.262162:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.262165:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.262168:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.262171:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262174:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59cb800 x1818247335526272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018438.262175:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018438.262183:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335526912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018438.262190:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018438.262193:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734018438.262193:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335526272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018438.262195:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018438.262198:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2702 00000100:00080000:0.0:1734018438.262200:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018438.262203:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734018438.262203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1734018438.262207:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734018438.262207:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335526272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018438.262216:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335527040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018438.262219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335526400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018438.262235:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59cad80 x1818247335526400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018438.262253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335526400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018438.262261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018438.262263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335526400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018438.262291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335526528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018438.262304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59cb100 x1818247335526528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018438.262321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335526528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018438.262335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018438.262338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335526528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018438.262347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335526656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018438.262358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59c8000 x1818247335526656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018438.262374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335526656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018438.262381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018438.262383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335526656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018438.262393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018438.262404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262410:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59c9500 x1818247335526784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018438.262420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335526784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018438.262428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018438.262430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018438.262439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018438.262451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59c9180 x1818247335526912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018438.262467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335526912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018438.262474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018438.262477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335526912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018438.262485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018438.262496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018438.262502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018438/real 1734018438] req@ffff8800b59ca300 x1818247335527040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018438.262512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335527040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018438.262519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018438.262521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018438.262524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527040:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018438.847956:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018439.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018439.849948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018440.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018440.852000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018441.213931:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018441.853976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018442.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018442.855960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018443.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018443.269830:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.269839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018443.269844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.269855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.269865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.269873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018443.269952:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335527168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018443.269986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.269991:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018443.269993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.269997:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.270001:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.270004:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018443.270014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018443.270027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335527296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018443.270037:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.270040:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018443.270041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.270045:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.270047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.270051:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018443.270055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018443.270064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335527424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018443.270073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.270077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018443.270078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.270082:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.270084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.270088:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018443.270092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018443.270100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335527552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018443.270110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.270113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018443.270114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.270118:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.270120:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.270124:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018443.270128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018443.270132:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335527168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018443.270136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335527680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018443.270143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.270146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018443.270148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.270152:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.270154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.270158:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018443.270163:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018443.270171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335527808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018443.270179:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018443.270182:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018443.270183:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018443.270186:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2707 00000100:00080000:0.0:1734018443.270189:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018443.270192:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018443.270195:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018443.270203:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335527936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018443.270227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59c9f80 x1818247335527168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335527168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335527168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018443.270313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335527296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018443.270331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59c8700 x1818247335527296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335527296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335527296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018443.270371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335527424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018443.270386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59ca300 x1818247335527424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335527424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270417:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335527424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018443.270457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335527552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018443.270471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59c9180 x1818247335527552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335527552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335527552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018443.270508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018443.270520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59c9500 x1818247335527680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335527680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018443.270556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018443.270569:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270575:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59c8000 x1818247335527808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270586:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335527808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270593:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270595:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018443.270604:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018443.270617:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018443.270623:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018443/real 1734018443] req@ffff8800b59cb100 x1818247335527936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018443.270633:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335527936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018443.270642:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018443.270644:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018443.270647:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335527936:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018443.857969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018444.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018444.859994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018445.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018445.861955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018446.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018446.863952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018447.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018447.865998:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018448.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018448.277879:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.277890:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018448.277895:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.277910:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.277922:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.277932:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018448.277981:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335528064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018448.278020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.278024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018448.278026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.278031:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.278036:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.278040:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018448.278052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018448.278064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335528192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018448.278075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.278078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018448.278080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.278084:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.278087:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.278091:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018448.278096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018448.278104:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335528320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018448.278113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.278116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018448.278117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.278121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.278123:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.278127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018448.278131:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018448.278139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335528448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018448.278141:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335528064:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018448.278155:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.278159:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018448.278160:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.278164:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.278166:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.278170:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018448.278174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018448.278181:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335528576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018448.278189:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.278192:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018448.278194:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.278197:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.278200:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.278203:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018448.278207:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018448.278217:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335528704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018448.278225:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018448.278228:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018448.278229:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018448.278233:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2712 00000100:00080000:0.0:1734018448.278237:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018448.278240:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018448.278245:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018448.278253:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335528832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018448.278257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335528192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018448.278281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335528320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018448.278309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335528448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018448.278326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335528576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018448.278342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335528704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018448.278357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335528832:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018448.868015:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018449.213902:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018449.869970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018450.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018450.872011:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018451.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734018451.284076:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018451.284133:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018451.284156:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018451.284201:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018451.284209:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018451.284218:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018451.284225:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018451.284232:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018451.284241:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018451.284316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59caa00 x1818247335528448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335528448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335528448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018451.284406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59cb480 x1818247335528576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335528576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335528576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018451.284464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59cbb80 x1818247335528704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335528704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335528704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018451.284530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59ca680 x1818247335528832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335528832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335528832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018451.284581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59cad80 x1818247335528064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284601:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335528064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284617:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335528064:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018451.284633:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59cb800 x1818247335528192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284651:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335528192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284665:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284669:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284674:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335528192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018451.284684:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018448/real 1734018451] req@ffff8800b59c9c00 x1818247335528320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018451.284970:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335528320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018451.284983:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018451.284988:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018451.284993:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335528320:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018451.873991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018452.213940:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018452.875965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018453.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018453.285862:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.285870:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018453.285874:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.285885:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.285895:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.285902:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018453.285939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335528960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018453.285973:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.285978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018453.285979:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.285984:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.285987:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.285991:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018453.286000:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018453.286011:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335529088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018453.286028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.286031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018453.286033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.286036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.286039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.286042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018453.286046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018453.286055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335529216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018453.286064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.286067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018453.286069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.286072:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.286074:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.286078:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018453.286082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018453.286090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335529344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018453.286099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.286103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018453.286104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.286108:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.286112:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.286115:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018453.286120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018453.286122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335528960:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018453.286129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335529472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018453.286137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.286140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018453.286141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.286145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.286147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.286150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018453.286155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018453.286162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335529600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018453.286170:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018453.286173:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018453.286174:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018453.286177:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2717 00000100:00080000:0.0:1734018453.286179:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018453.286182:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018453.286186:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018453.286193:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335529728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018453.286219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286248:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59c8a80 x1818247335528960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335528960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335528960:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018453.286302:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018453.286320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59c8e00 x1818247335529088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286340:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335529088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286351:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018453.286360:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018453.286375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59c9c00 x1818247335529216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335529216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018453.286438:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018453.286452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286459:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59cb800 x1818247335529344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335529344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018453.286490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335529472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018453.286503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59cad80 x1818247335529472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335529472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286526:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335529472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018453.286538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335529600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018453.286550:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59ca680 x1818247335529600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335529600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335529600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018453.286586:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335529728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018453.286598:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018453.286604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018453/real 1734018453] req@ffff8800b59cbb80 x1818247335529728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018453.286614:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335529728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018453.286621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018453.286623:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018453.286626:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335529728:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018453.877999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018454.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018454.879993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018455.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018455.881975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018456.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018456.883999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018457.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018457.885941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018458.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018458.293826:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.293834:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018458.293838:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.293849:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.293858:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.293866:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018458.293903:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335529856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018458.293938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.293942:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018458.293944:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.293948:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.293951:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.293955:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018458.293965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018458.293975:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335529984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018458.293985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.293988:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018458.293991:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.293995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.293997:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.294000:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018458.294005:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018458.294013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335530112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018458.294021:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.294024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018458.294026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.294029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.294031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.294035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018458.294039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018458.294047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335530240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018458.294055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.294058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018458.294059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.294063:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.294065:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.294069:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018458.294073:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018458.294080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335530368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018458.294090:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.294093:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018458.294095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.294099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.294101:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.294104:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294104:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335529856:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018458.294108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018458.294116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335530496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018458.294123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018458.294126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018458.294128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018458.294131:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2722 00000100:00080000:0.0:1734018458.294133:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018458.294137:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018458.294141:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018458.294150:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335530624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018458.294200:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59cb480 x1818247335529856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335529856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294269:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335529856:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018458.294290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018458.294309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59caa00 x1818247335529984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335529984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294340:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335529984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018458.294350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335530112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018458.294364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59cb100 x1818247335530112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335530112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294391:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335530112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018458.294400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335530240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018458.294414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294420:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59c8000 x1818247335530240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294432:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335530240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335530240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018458.294481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335530368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018458.294494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59c9500 x1818247335530368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335530368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335530368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018458.294530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335530496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018458.294542:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59c9180 x1818247335530496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294558:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335530496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335530496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018458.294577:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335530624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018458.294589:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018458.294595:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018458/real 1734018458] req@ffff8800b59ca300 x1818247335530624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018458.294605:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335530624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018458.294612:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018458.294614:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018458.294617:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335530624:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018458.887919:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018459.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018459.889988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018460.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018460.891923:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018461.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018461.893928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018462.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018462.895972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018463.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018463.301813:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.301822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018463.301826:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.301837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.301846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018463.301853:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018463.301889:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335530752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018463.301926:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.301930:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018463.301932:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.301936:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.301940:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018463.301943:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018463.301953:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018463.301964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335530880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018463.301974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.301977:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018463.301979:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.301982:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.301985:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018463.301988:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018463.301992:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018463.302002:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335531008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018463.302012:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.302016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018463.302017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.302021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.302023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018463.302027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018463.302031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018463.302038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335531136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018463.302048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.302051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018463.302052:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.302056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.302059:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018463.302061:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335530752:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018463.302062:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018463.302066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018463.302074:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335531264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018463.302082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.302084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018463.302086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.302090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.302093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018463.302096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018463.302101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018463.302108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335531392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018463.302115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018463.302118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018463.302120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018463.302123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2727 00000100:00080000:0.0:1734018463.302127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018463.302130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018463.302134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018463.302142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335531520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018463.302157:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59c8700 x1818247335530752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335530752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335530752:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018463.302246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335530880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018463.302265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59c9f80 x1818247335530880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335530880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335530880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018463.302305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018463.302320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302326:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59ca300 x1818247335531008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302336:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335531008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018463.302382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018463.302396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302402:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59c9180 x1818247335531136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335531136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018463.302434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335531264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018463.302446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59c9500 x1818247335531264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335531264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335531264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018463.302480:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335531392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018463.302493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59c8000 x1818247335531392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335531392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302516:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335531392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018463.302529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335531520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018463.302542:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018463.302548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018463/real 1734018463] req@ffff8800b59cb100 x1818247335531520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018463.302557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335531520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018463.302565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018463.302567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018463.302570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335531520:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018463.897941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018464.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018464.899982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018465.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018465.901972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018466.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018466.903965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018467.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018467.905936:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018468.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018468.309832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.309841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018468.309845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.309855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.309865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.309872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018468.309911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335531648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018468.309945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.309949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018468.309951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.309956:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.309959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.309963:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018468.309972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018468.309982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335531776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018468.309992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.309995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018468.309997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.310000:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.310003:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.310006:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018468.310011:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018468.310018:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335531904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018468.310026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.310030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018468.310032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.310036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.310039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.310042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018468.310046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018468.310054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335532032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018468.310062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.310065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018468.310067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.310070:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.310072:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.310076:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018468.310080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018468.310088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335532160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018468.310091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335531648:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018468.310097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.310100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018468.310101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.310105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.310107:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.310110:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018468.310115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018468.310122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335532288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018468.310131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018468.310134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018468.310136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018468.310139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2732 00000100:00080000:0.0:1734018468.310141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018468.310144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018468.310148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018468.310157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335532416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018468.310186:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59caa00 x1818247335531648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335531648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335531648:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018468.310270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018468.310289:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59cb480 x1818247335531776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335531776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018468.310329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018468.310344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59cbb80 x1818247335531904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335531904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335531904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018468.310379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018468.310393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59ca680 x1818247335532032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310417:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335532032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310425:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018468.310464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335532160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018468.310478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59cad80 x1818247335532160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335532160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335532160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018468.310551:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335532288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018468.310566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59cb800 x1818247335532288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310584:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335532288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310592:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310595:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335532288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018468.310605:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335532416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018468.310618:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018468.310624:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018468/real 1734018468] req@ffff8800b59c9c00 x1818247335532416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018468.310634:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335532416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018468.310642:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018468.310644:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018468.310647:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335532416:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018468.907981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018469.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018469.909915:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018470.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018470.911899:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018471.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018471.913964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018472.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018472.915948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018473.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018473.317833:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.317841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018473.317845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.317855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.317865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.317872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018473.317909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335532544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018473.317943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.317948:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018473.317950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.317955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.317958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.317962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018473.317972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018473.317982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335532672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018473.317992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.317995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018473.317997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.318001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.318003:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.318007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018473.318011:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018473.318021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335532800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018473.318030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.318033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018473.318034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.318038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.318040:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.318043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018473.318047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018473.318055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335532928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018473.318065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.318069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018473.318070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.318074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.318076:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.318080:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335532544:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018473.318084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018473.318091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335533056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018473.318099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.318102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018473.318103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.318107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.318109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.318112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018473.318116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018473.318123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335533184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018473.318131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018473.318134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018473.318135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018473.318138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2737 00000100:00080000:0.0:1734018473.318141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018473.318144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018473.318148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018473.318156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335533312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018473.318175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59c8e00 x1818247335532544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335532544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335532544:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018473.318262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018473.318282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59c8a80 x1818247335532672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335532672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018473.318323:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018473.318338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59c9c00 x1818247335532800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335532800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318364:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318367:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018473.318448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018473.318468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59cb800 x1818247335532928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335532928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335532928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018473.318509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018473.318522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59cad80 x1818247335533056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335533056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018473.318558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018473.318570:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318576:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59ca680 x1818247335533184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318586:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335533184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318594:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318597:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318600:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018473.318607:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018473.318621:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018473.318626:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018473/real 1734018473] req@ffff8800b59cbb80 x1818247335533312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018473.318636:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335533312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018473.318643:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018473.318646:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018473.318649:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533312:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018473.917939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018474.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018474.919988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018475.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018475.921949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018476.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018476.923976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018477.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018477.925966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018478.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018478.325834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.325843:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018478.325847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.325857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.325867:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.325874:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018478.325910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335533440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018478.325945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.325949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018478.325951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.325955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.325958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.325962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018478.325971:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018478.325982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335533568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018478.325992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.325996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018478.325998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.326001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.326004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.326007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018478.326012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018478.326019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335533696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018478.326029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.326032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018478.326033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.326037:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.326039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.326042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018478.326046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018478.326054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335533824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018478.326062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.326065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018478.326067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.326071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.326073:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.326077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018478.326081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018478.326089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335533952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018478.326089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335533440:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018478.326098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.326101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018478.326103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.326106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.326108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.326112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018478.326116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018478.326123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335534080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018478.326130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018478.326133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018478.326135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018478.326138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2742 00000100:00080000:0.0:1734018478.326140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018478.326143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018478.326147:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018478.326156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335534208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018478.326184:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59cb480 x1818247335533440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326238:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335533440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335533440:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018478.326271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335533568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018478.326291:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59caa00 x1818247335533568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335533568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335533568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018478.326367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335533696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018478.326386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326394:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59cb100 x1818247335533696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335533696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335533696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018478.326425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335533824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018478.326438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59c8000 x1818247335533824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335533824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335533824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018478.326504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018478.326517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59c9500 x1818247335533952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335533952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326541:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326544:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335533952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018478.326553:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018478.326566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326572:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59c9180 x1818247335534080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326581:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335534080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326589:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018478.326602:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018478.326615:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018478.326621:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018478/real 1734018478] req@ffff8800b59ca300 x1818247335534208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018478.326631:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335534208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018478.326638:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018478.326640:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018478.326643:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534208:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018478.927954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018479.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018479.929939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018480.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018480.931948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018481.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018481.933995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018482.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018482.935955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018483.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018483.333842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.333850:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018483.333854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.333864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.333873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.333881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018483.333917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335534336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018483.333950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.333955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018483.333957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.333961:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.333964:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.333968:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018483.333977:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018483.333988:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335534464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018483.333997:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.334001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018483.334003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.334007:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.334009:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.334013:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018483.334017:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018483.334026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335534592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018483.334035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.334038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018483.334039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.334043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.334045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.334048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018483.334052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018483.334060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335534720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018483.334069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.334072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018483.334073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.334077:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.334080:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.334083:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018483.334087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018483.334094:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335534848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018483.334102:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335534336:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018483.334105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.334108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018483.334110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.334114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.334116:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.334119:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018483.334123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018483.334130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335534976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018483.334138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018483.334141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018483.334143:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018483.334146:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2747 00000100:00080000:0.0:1734018483.334148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018483.334151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018483.334155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018483.334162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335535104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018483.334194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59c9f80 x1818247335534336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335534336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335534336:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018483.334315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335534464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018483.334340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59c8700 x1818247335534464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335534464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335534464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018483.334380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335534592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018483.334395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334402:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59ca300 x1818247335534592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335534592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335534592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018483.334431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335534720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018483.334445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59c9180 x1818247335534720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335534720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335534720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018483.334517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018483.334531:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59c9500 x1818247335534848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334547:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335534848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334555:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018483.334566:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018483.334579:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334585:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59c8000 x1818247335534976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334595:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335534976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334602:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334609:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335534976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018483.334615:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335535104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018483.334628:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018483.334634:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018483/real 1734018483] req@ffff8800b59cb100 x1818247335535104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018483.334644:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335535104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018483.334652:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018483.334654:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018483.334658:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335535104:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018483.937985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018484.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018484.939965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018485.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018485.941930:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018486.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018486.943970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018487.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018487.945935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018488.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018488.341839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.341848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018488.341852:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.341862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.341871:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018488.341879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018488.341914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335535232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018488.341948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.341952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018488.341954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.341958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.341962:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018488.341965:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018488.341975:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018488.341986:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335535360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018488.341995:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.341999:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018488.342000:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.342004:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.342006:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018488.342010:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018488.342014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018488.342022:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335535488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018488.342031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.342034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018488.342036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.342039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.342042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018488.342045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018488.342049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018488.342056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335535616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018488.342064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.342067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018488.342069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.342072:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.342075:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018488.342078:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018488.342082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018488.342089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335535744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018488.342098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.342101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018488.342102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.342106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.342109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018488.342111:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335535232:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018488.342112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018488.342117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018488.342124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335535872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018488.342132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018488.342135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018488.342136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018488.342139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2752 00000100:00080000:0.0:1734018488.342142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018488.342172:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018488.342177:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018488.342188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335536000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018488.342234:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342266:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59caa00 x1818247335535232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342288:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335535232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335535232:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018488.342323:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335535360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018488.342343:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59cb480 x1818247335535360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335535360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335535360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018488.342384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335535488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018488.342398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59cbb80 x1818247335535488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335535488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335535488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018488.342434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335535616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018488.342447:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59ca680 x1818247335535616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335535616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335535616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018488.342514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335535744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018488.342527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59cad80 x1818247335535744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335535744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342551:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335535744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018488.342563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335535872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018488.342575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59cb800 x1818247335535872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335535872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342598:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335535872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018488.342611:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018488.342624:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018488.342630:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018488/real 1734018488] req@ffff8800b59c9c00 x1818247335536000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018488.342640:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335536000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018488.342647:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018488.342649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018488.342652:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536000:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018488.947933:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018489.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018489.949970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018490.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018490.951955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018491.213927:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018491.953967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018492.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018492.955935:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018493.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018493.349809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.349817:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018493.349822:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.349832:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.349841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.349848:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018493.349882:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335536128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018493.349923:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.349927:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018493.349929:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.349933:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.349936:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.349940:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018493.349949:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018493.349959:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335536256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018493.349968:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.349971:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018493.349973:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.349977:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.349979:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.349983:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018493.349987:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018493.349996:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335536384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018493.350005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.350008:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018493.350009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.350013:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.350015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.350019:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018493.350023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018493.350031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335536512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018493.350042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.350045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018493.350046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.350049:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.350052:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.350055:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018493.350059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018493.350100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335536640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018493.350102:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335536128:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018493.350108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.350111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018493.350113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.350116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.350119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.350122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018493.350127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018493.350134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335536768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018493.350142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018493.350145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018493.350147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018493.350150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2757 00000100:00080000:0.0:1734018493.350153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018493.350156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018493.350160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018493.350167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335536896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018493.350199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59c8a80 x1818247335536128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335536128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335536128:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018493.350288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335536256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018493.350307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59c8e00 x1818247335536256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335536256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335536256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018493.350347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335536384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018493.350361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59c9c00 x1818247335536384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335536384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350387:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350390:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335536384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018493.350424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335536512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018493.350438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59cb800 x1818247335536512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335536512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335536512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018493.350476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018493.350489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59cad80 x1818247335536640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335536640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018493.350523:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018493.350535:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350541:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59ca680 x1818247335536768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350551:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335536768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018493.350572:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018493.350585:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018493.350590:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018493/real 1734018493] req@ffff8800b59cbb80 x1818247335536896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018493.350600:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335536896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018493.350607:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018493.350610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018493.350613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335536896:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018493.958040:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018494.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018494.960023:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018495.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018495.962000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018496.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018496.963920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018497.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018497.965946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018498.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018498.357838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.357847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018498.357851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.357862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.357871:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.357879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018498.357916:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335537024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018498.357950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.357954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018498.357956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.357961:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.357964:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.357967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018498.358018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018498.358031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335537152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018498.358042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.358045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018498.358047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.358051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.358054:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.358057:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018498.358062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018498.358070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335537280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018498.358079:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.358082:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018498.358084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.358087:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.358090:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.358093:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018498.358097:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018498.358106:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335537408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018498.358113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335537024:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018498.358115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.358118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018498.358120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.358124:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.358127:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.358131:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018498.358135:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018498.358142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335537536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018498.358150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.358153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018498.358155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.358158:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.358160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.358164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018498.358167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018498.358174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335537664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018498.358182:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018498.358185:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018498.358186:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018498.358189:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2762 00000100:00080000:0.0:1734018498.358193:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018498.358196:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018498.358199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018498.358208:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335537792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018498.358210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59cb480 x1818247335537024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335537024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335537024:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018498.358292:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335537152:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018498.358312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358319:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59caa00 x1818247335537152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335537152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358339:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335537152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018498.358352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335537280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018498.358367:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59cb100 x1818247335537280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335537280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335537280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018498.358428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335537408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018498.358442:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59c8000 x1818247335537408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335537408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335537408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018498.358487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335537536:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018498.358499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59c9500 x1818247335537536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335537536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335537536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018498.358534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335537664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018498.358546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59c9180 x1818247335537664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358561:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335537664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358575:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335537664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018498.358580:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335537792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018498.358593:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018498.358598:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018498/real 1734018498] req@ffff8800b59ca300 x1818247335537792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018498.358608:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335537792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018498.358616:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018498.358618:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018498.358621:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335537792:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018498.968015:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018499.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018499.969981:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018500.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018500.972000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018501.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018501.974016:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018502.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018502.975976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018503.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018503.365809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.365818:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018503.365822:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.365832:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.365841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.365848:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018503.365885:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335537920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018503.365955:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.365959:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018503.365961:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.365966:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.365970:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.365974:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018503.365984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018503.365996:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335538048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018503.366006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.366009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018503.366011:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.366015:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.366017:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.366021:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018503.366026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018503.366036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335538176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018503.366045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.366048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018503.366050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.366053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.366055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.366059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018503.366062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018503.366070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335538304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018503.366080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.366084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018503.366085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.366089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.366092:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.366095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335537920:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018503.366099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018503.366107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335538432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018503.366115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.366117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018503.366119:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.366124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.366126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.366129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018503.366133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018503.366141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335538560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018503.366149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018503.366151:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018503.366153:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018503.366156:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2767 00000100:00080000:0.0:1734018503.366159:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018503.366162:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018503.366166:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018503.366173:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335538688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018503.366190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59c8700 x1818247335537920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335537920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366257:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335537920:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018503.366278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018503.366298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59c9f80 x1818247335538048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335538048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018503.366338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018503.366354:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366360:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59ca300 x1818247335538176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335538176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366378:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018503.366416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018503.366430:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59c9180 x1818247335538304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335538304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366456:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018503.366467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335538432:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018503.366480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59c9500 x1818247335538432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335538432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335538432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018503.366514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335538560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018503.366526:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59c8000 x1818247335538560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335538560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335538560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018503.366563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335538688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018503.366576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018503.366582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018503/real 1734018503] req@ffff8800b59cb100 x1818247335538688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018503.366592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335538688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018503.366599:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018503.366602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018503.366605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335538688:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018503.977963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018504.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018504.980054:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018505.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018505.981941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018506.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018506.983990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018507.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018507.985995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018508.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018508.373804:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.373846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018508.373851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.373862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.373872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.373879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018508.373915:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335538816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018508.373958:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.373962:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018508.373964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.373968:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.373972:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.373975:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018508.373985:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018508.373995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335538944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018508.374004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.374007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018508.374009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.374012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.374015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.374018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018508.374022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018508.374030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335539072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018508.374038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.374042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018508.374043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.374047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.374049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.374053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018508.374058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018508.374066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335539200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018508.374076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.374079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018508.374080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.374084:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.374086:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.374090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018508.374094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018508.374099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335538816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018508.374102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335539328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018508.374110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.374113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018508.374114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.374117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.374120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.374123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018508.374127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018508.374134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335539456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018508.374142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018508.374146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018508.374148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018508.374151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2772 00000100:00080000:0.0:1734018508.374153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018508.374156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018508.374160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018508.374169:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335539584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018508.374194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59caa00 x1818247335538816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335538816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335538816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018508.374281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018508.374300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59cb480 x1818247335538944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335538944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335538944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018508.374340:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018508.374355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374361:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59cbb80 x1818247335539072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335539072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018508.374418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018508.374432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59ca680 x1818247335539200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335539200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018508.374470:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335539328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018508.374482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59cad80 x1818247335539328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335539328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335539328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018508.374516:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335539456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018508.374528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59cb800 x1818247335539456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335539456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374551:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335539456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018508.374563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335539584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018508.374575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018508.374581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018508/real 1734018508] req@ffff8800b59c9c00 x1818247335539584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018508.374591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335539584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018508.374598:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018508.374600:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018508.374603:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335539584:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018508.988000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018509.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018509.989973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018510.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018510.992034:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018511.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018511.993996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018512.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018512.995958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018513.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018513.381827:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.381836:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018513.381841:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.381851:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.381860:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.381868:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018513.381904:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335539712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018513.381938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.381943:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018513.381945:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.381949:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.381952:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.381956:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018513.381966:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018513.381976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335539840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018513.381987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.381990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018513.381992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.381996:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.381998:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.382002:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018513.382006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018513.382016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335539968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018513.382025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.382028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018513.382030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.382033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.382035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.382039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018513.382043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018513.382051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335540096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018513.382059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.382062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018513.382064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.382067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.382070:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.382074:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018513.382079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018513.382087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335540224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018513.382091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335539712:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018513.382096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.382099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018513.382100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.382104:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.382106:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.382109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018513.382114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018513.382122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335540352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018513.382129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018513.382132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018513.382134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018513.382137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2777 00000100:00080000:0.0:1734018513.382139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018513.382142:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018513.382146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018513.382153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335540480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018513.382220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018513.382245:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018513.382263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018513.382290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335540224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018513.382308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335540352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018513.382324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335540480:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018513.997986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018514.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018514.999972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018515.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018516.001965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018516.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734018516.388124:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018516.388186:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018516.388211:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018516.388260:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018516.388271:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018516.388279:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018516.388286:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018516.388294:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018516.388300:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018516.388386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59cad80 x1818247335540224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.388420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335540224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.388434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.388439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.388445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335540224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018516.388453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59ca680 x1818247335540352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.388466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335540352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.388474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.388477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.388481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335540352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018516.388488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59cbb80 x1818247335540480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.388499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335540480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.388507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.388510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.388513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335540480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018516.388520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59c8e00 x1818247335539712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.388535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335539712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.388543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.388546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.388549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335539712:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018516.388556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59c8a80 x1818247335539840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.388567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335539840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.388575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.388579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.388584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018516.388596:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59c9c00 x1818247335539968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.388613:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335539968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.388624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.388628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.388633:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335539968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018516.388644:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018513/real 1734018516] req@ffff8800b59cb800 x1818247335540096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018516.389047:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335540096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018516.389063:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018516.389068:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018516.389073:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540096:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018517.003994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018517.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018518.005977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018518.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018518.389800:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.389808:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018518.389813:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.389823:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.389832:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.389840:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018518.389936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335540608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018518.389969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.389973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018518.389975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.389980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.389983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.389987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018518.389997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018518.390007:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335540736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018518.390018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.390021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018518.390023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.390026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.390029:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.390032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018518.390038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018518.390046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335540864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018518.390058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.390061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018518.390062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.390066:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.390068:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.390072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018518.390076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018518.390083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335540608:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018518.390084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335540992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018518.390093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.390096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018518.390097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.390101:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.390103:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.390107:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018518.390111:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018518.390118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335541120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018518.390127:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.390129:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018518.390132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.390135:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.390138:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.390141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018518.390145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018518.390154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335541248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018518.390162:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018518.390164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018518.390166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018518.390169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2782 00000100:00080000:0.0:1734018518.390172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018518.390175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018518.390179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018518.390179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018518.390187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335541376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018518.390211:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59cb480 x1818247335540608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335540608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335540608:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018518.390264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018518.390284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018518.390290:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59caa00 x1818247335540736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335540736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018518.390353:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018518.390368:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018518.390375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59cb100 x1818247335540864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335540864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018518.390406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018518.390418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018518.390425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59c8000 x1818247335540992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335540992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335540992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018518.390456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335541120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018518.390470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018518.390477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59c9500 x1818247335541120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335541120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335541120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018518.390505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335541248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018518.390518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018518.390524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59c9180 x1818247335541248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335541248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335541248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018518.390554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335541376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018518.390568:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018518.390574:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018518/real 1734018518] req@ffff8800b59ca300 x1818247335541376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018518.390585:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335541376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018518.390593:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018518.390596:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018518.390599:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335541376:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018519.007895:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018519.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018520.009982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018520.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018521.011950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018521.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018522.013973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018522.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018523.015985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018523.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018523.397804:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.397812:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018523.397817:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.397826:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.397835:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.397842:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018523.397877:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335541504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018523.397919:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.397923:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018523.397924:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.397929:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.397932:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.397936:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018523.397945:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018523.397956:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335541632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018523.397965:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.397968:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018523.397970:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.397973:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.397976:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.397979:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018523.397983:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018523.397993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335541760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018523.398002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.398005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018523.398006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.398009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.398012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.398016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018523.398020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018523.398028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335541888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018523.398038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.398041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018523.398042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.398046:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.398048:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.398051:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018523.398056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018523.398057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335541504:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018523.398063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335542016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018523.398071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.398074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018523.398075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.398079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.398081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.398084:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018523.398088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018523.398096:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335542144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018523.398104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018523.398107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018523.398109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018523.398112:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2787 00000100:00080000:0.0:1734018523.398115:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018523.398118:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018523.398122:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018523.398129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335542272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018523.398147:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398177:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59c9f80 x1818247335541504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398199:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335541504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398213:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335541504:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018523.398230:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335541632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018523.398248:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59c8700 x1818247335541632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335541632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398275:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398282:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335541632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018523.398288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335541760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018523.398302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59ca300 x1818247335541760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335541760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335541760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018523.398366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335541888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018523.398380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59c9180 x1818247335541888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335541888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335541888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018523.398418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018523.398431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59c9500 x1818247335542016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398447:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335542016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018523.398466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018523.398478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59c8000 x1818247335542144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335542144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018523.398514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018523.398527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018523.398533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018523/real 1734018523] req@ffff8800b59cb100 x1818247335542272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018523.398544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335542272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018523.398587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018523.398590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018523.398594:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542272:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018524.017961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018524.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018525.019917:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018525.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018526.021955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018526.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018527.023951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018527.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018528.025947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018528.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018528.405836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.405844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018528.405848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.405859:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.405868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.405875:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018528.405911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335542400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018528.405946:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.405950:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018528.405952:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.405957:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.405960:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.405964:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018528.405973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018528.405984:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335542528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018528.405995:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.405998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018528.405999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.406003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.406005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.406009:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018528.406013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018528.406021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335542656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018528.406029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.406032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018528.406034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.406037:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.406040:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.406043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018528.406047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018528.406054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335542784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018528.406062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.406065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018528.406067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.406071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.406074:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.406077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018528.406081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018528.406089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335542912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018528.406094:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335542400:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018528.406099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.406102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018528.406103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.406107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.406110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.406113:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018528.406117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018528.406124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335543040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018528.406131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018528.406134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018528.406136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018528.406139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2792 00000100:00080000:0.0:1734018528.406141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018528.406144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018528.406148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018528.406157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335543168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018528.406191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406223:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59caa00 x1818247335542400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335542400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335542400:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018528.406276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335542528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018528.406295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59cb480 x1818247335542528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335542528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335542528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018528.406343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335542656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018528.406358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59cb800 x1818247335542656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335542656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335542656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018528.406395:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335542784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018528.406408:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406413:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59c9c00 x1818247335542784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335542784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406431:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406434:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335542784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018528.406511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542912:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018528.406531:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59c8a80 x1818247335542912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406551:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335542912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335542912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018528.406571:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018528.406583:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406589:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59c8e00 x1818247335543040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406599:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335543040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018528.406619:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018528.406631:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018528.406637:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018528/real 1734018528] req@ffff8800b59cbb80 x1818247335543168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018528.406648:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335543168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018528.406656:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018528.406658:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018528.406661:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543168:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018529.027986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018529.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018530.029949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018530.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018531.031972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018531.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018532.034019:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018532.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018533.035998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018533.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018533.413809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.413819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018533.413824:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.413837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.413848:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.413857:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018533.413903:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335543296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018533.413941:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.413946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018533.413948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.413953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.413956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.413960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018533.413972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018533.413983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335543424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018533.413993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.413996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018533.413998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.414002:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.414005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.414008:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018533.414015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018533.414026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335543552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018533.414035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.414038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018533.414040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.414043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.414046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.414049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018533.414054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018533.414058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335543296:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018533.414062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335543680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018533.414071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.414074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018533.414076:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.414080:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.414082:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.414086:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018533.414090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018533.414098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335543808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018533.414107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.414110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018533.414112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.414116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.414119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.414123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018533.414127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018533.414135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335543936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018533.414142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018533.414145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1734018533.414145:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018533.414147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018533.414150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2797 00000100:00080000:0.0:1734018533.414153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018533.414156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018533.414160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018533.414168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335544064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018533.414173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59ca680 x1818247335543296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414194:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335543296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414206:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414209:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414214:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335543296:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018533.414222:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335543424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018533.414239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018533.414246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59cad80 x1818247335543424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335543424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335543424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018533.414279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335543552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018533.414293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018533.414299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59cbb80 x1818247335543552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414309:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335543552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335543552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018533.414351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335543680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018533.414364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018533.414371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59c8e00 x1818247335543680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335543680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335543680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018533.414435:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018533.414451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018533.414458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59c8a80 x1818247335543808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335543808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018533.414488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018533.414499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018533.414505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59c9c00 x1818247335543936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335543936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335543936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018533.414534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018533.414547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018533.414552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018533/real 1734018533] req@ffff8800b59cb800 x1818247335544064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018533.414563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335544064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018533.414570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018533.414572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018533.414575:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544064:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018534.037956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018534.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018535.039988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018535.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018536.041959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018536.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018537.043925:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018537.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018538.045977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018538.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018538.421818:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.421827:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018538.421832:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.421842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.421851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.421859:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018538.421893:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335544192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018538.421935:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.421939:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018538.421941:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.421945:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.421948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.421952:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018538.421962:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018538.421972:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335544320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018538.421981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.421985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018538.421986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.421990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.421992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.421995:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018538.422000:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018538.422007:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335544448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018538.422016:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.422019:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018538.422020:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.422024:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.422026:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.422030:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018538.422034:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018538.422042:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335544576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018538.422050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.422053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018538.422054:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.422057:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.422060:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.422063:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018538.422066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018538.422074:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335544704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018538.422082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335544192:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018538.422084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.422087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018538.422088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.422092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.422094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.422098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018538.422102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018538.422110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335544832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018538.422117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018538.422121:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018538.422123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018538.422126:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2802 00000100:00080000:0.0:1734018538.422128:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018538.422132:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018538.422136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018538.422144:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335544960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018538.422176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59cb480 x1818247335544192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335544192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422253:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335544192:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018538.422261:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335544320:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018538.422280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59caa00 x1818247335544320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335544320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335544320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018538.422356:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335544448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018538.422373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422380:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59cb100 x1818247335544448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335544448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335544448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018538.422411:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335544576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018538.422424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59c8000 x1818247335544576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335544576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335544576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018538.422489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544704:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018538.422504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59c9500 x1818247335544704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335544704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018538.422541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018538.422553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59c9180 x1818247335544832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335544832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422576:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018538.422588:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018538.422600:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018538.422606:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018538/real 1734018538] req@ffff8800b59ca300 x1818247335544960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018538.422618:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335544960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018538.422626:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018538.422628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018538.422631:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335544960:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018539.047934:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018539.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018540.049980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018540.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018541.051992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018541.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018542.053962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018542.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018543.055960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018543.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018543.429819:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.429828:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018543.429832:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.429842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.429851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.429859:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018543.429895:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335545088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018543.429930:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.429934:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018543.429936:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.429941:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.429944:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.429947:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018543.429957:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018543.429967:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335545216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018543.429978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.429981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018543.429983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.429986:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.429989:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.429992:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018543.429996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018543.430005:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335545344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018543.430014:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.430017:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018543.430019:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.430022:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.430025:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.430028:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018543.430032:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018543.430040:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335545472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018543.430048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.430051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018543.430053:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.430056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.430060:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.430063:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018543.430067:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018543.430075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335545600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018543.430077:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335545088:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018543.430085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.430087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018543.430089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.430092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.430095:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.430098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018543.430102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018543.430109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335545728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018543.430117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018543.430120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018543.430121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018543.430124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2807 00000100:00080000:0.0:1734018543.430127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018543.430130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018543.430133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018543.430141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335545856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018543.430171:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430202:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59c8700 x1818247335545088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335545088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335545088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018543.430285:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335545216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018543.430306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59c9f80 x1818247335545216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335545216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430344:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335545216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018543.430354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335545344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018543.430369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59ca300 x1818247335545344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335545344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335545344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018543.430406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335545472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018543.430419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59c9180 x1818247335545472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335545472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335545472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018543.430484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335545600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018543.430497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59c9500 x1818247335545600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335545600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335545600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018543.430534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335545728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018543.430546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430551:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59c8000 x1818247335545728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430561:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335545728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335545728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018543.430581:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335545856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018543.430593:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018543.430599:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018543/real 1734018543] req@ffff8800b59cb100 x1818247335545856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018543.430610:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335545856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018543.430618:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018543.430620:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018543.430623:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335545856:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018544.057962:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018544.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018545.059961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018545.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018546.061920:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018546.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018547.063989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018547.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018548.065971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018548.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018548.437833:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.437843:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018548.437847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.437857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.437866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.437874:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018548.437909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335545984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018548.437943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.437947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018548.437949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.437953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.437956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.437960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018548.437970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018548.437980:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335546112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018548.437990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.437993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018548.437995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.437998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.438001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.438004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018548.438010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018548.438017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335546240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018548.438026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.438029:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018548.438030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.438034:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.438036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.438040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018548.438044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018548.438051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335546368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018548.438059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.438062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018548.438064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.438067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.438070:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.438073:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018548.438077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018548.438084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335546496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018548.438093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.438096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018548.438098:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.438102:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.438104:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.438108:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018548.438112:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018548.438113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335545984:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018548.438119:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335546624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018548.438163:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018548.438166:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018548.438167:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018548.438171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2812 00000100:00080000:0.0:1734018548.438174:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018548.438177:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018548.438182:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018548.438192:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335546752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018548.438257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59caa00 x1818247335545984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335545984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335545984:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018548.438343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335546112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018548.438362:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59cb480 x1818247335546112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335546112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438389:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335546112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018548.438402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335546240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018548.438418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59cb800 x1818247335546240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335546240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335546240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018548.438454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335546368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018548.438467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59c9c00 x1818247335546368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335546368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335546368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018548.438531:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335546496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018548.438545:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59c8a80 x1818247335546496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335546496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335546496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018548.438582:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335546624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018548.438594:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438599:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59c8e00 x1818247335546624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438609:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335546624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438619:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335546624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018548.438628:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335546752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018548.438639:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018548.438645:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018548/real 1734018548] req@ffff8800b59cbb80 x1818247335546752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018548.438656:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335546752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018548.438664:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018548.438667:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018548.438670:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335546752:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018549.067951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018549.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018550.070010:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018550.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018551.071973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018551.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018552.073989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018552.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018553.075949:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018553.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018553.445773:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.445782:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018553.445786:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.445797:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.445806:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.445813:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018553.445848:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335546880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018553.445888:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.445892:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018553.445894:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.445898:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.445901:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.445905:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018553.445914:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018553.445925:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335547008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018553.445934:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.445937:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018553.445939:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.445942:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.445945:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.445949:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018553.445953:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018553.445962:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335547136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018553.445971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.445974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018553.445976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.445980:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.445982:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.445986:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018553.445990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018553.445998:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335547264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018553.446008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.446011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018553.446012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.446016:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.446018:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.446022:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446022:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335546880:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018553.446026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018553.446033:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335547392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018553.446072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.446075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018553.446077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.446081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.446084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.446087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018553.446091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018553.446100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335547520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018553.446108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018553.446112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018553.446114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018553.446117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2817 00000100:00080000:0.0:1734018553.446120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018553.446123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018553.446127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018553.446135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335547648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018553.446143:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446174:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59cad80 x1818247335546880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446196:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335546880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446209:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446212:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446218:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335546880:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018553.446226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018553.446245:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59ca680 x1818247335547008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335547008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446279:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018553.446286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018553.446301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59cbb80 x1818247335547136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335547136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446327:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018553.446361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018553.446375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59c8e00 x1818247335547264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335547264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018553.446414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335547392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018553.446427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59c8a80 x1818247335547392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335547392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335547392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018553.446461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335547520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018553.446473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59c9c00 x1818247335547520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335547520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335547520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018553.446508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335547648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018553.446520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018553.446526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018553/real 1734018553] req@ffff8800b59cb800 x1818247335547648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018553.446538:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335547648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018553.446546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018553.446548:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018553.446551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335547648:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018554.077887:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018554.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018555.079982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018555.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018556.082051:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018556.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018557.084044:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018557.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018558.086048:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018558.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018558.453815:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.453827:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018558.453833:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.453847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00080000:0.0:1734018558.453859:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.453870:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018558.453919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335547776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018558.453995:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.454000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018558.454002:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.454007:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00080000:0.0:1734018558.454011:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.454015:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018558.454027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018558.454040:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335547904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018558.454052:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.454055:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018558.454057:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.454060:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00080000:0.0:1734018558.454063:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.454067:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018558.454071:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018558.454080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335548032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018558.454082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335547776:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018558.454089:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.454092:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018558.454094:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.454097:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00080000:0.0:1734018558.454100:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.454103:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018558.454108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018558.454117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335548160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018558.454125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.454129:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018558.454130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.454134:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00080000:0.0:1734018558.454137:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.454141:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018558.454146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018558.454154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335548288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018558.454161:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.454164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018558.454166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.454170:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00080000:0.0:1734018558.454172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.454176:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734018558.454177:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734018558.454180:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018558.454188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335548416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018558.454196:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018558.454199:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018558.454200:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018558.454204:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2822 00000100:00100000:3.0:1734018558.454206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59cb480 x1818247335547776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018558.454207:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018558.454210:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018558.454215:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018558.454224:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335548544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018558.454227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335547776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454248:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335547776:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018558.454255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018558.454273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018558.454280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59caa00 x1818247335547904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018558.454292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335547904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335547904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018558.454342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018558.454357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018558.454363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59cb100 x1818247335548032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018558.454374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335548032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018558.454393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018558.454406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018558.454411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59c8000 x1818247335548160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018558.454421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335548160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018558.454442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335548288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018558.454455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018558.454462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59c9500 x1818247335548288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018558.454472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335548288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335548288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018558.454491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335548416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018558.454504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018558.454510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59c9180 x1818247335548416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018558.454519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335548416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335548416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018558.454537:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335548544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018558.454549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018558.454555:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018558/real 1734018558] req@ffff8800b59ca300 x1818247335548544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018558.454566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335548544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018558.454574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018558.454576:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018558.454579:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335548544:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018559.088010:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018559.213787:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018560.090014:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018560.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018561.091995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018561.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018562.093989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018562.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018563.095951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018563.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018563.461843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.461853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018563.461857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018563.461867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.461912:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.461920:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018563.461955:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335548672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018563.461988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.461993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018563.461995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018563.461999:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.462002:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.462006:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018563.462015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018563.462025:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335548800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018563.462035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.462038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018563.462040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018563.462044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.462046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.462049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018563.462055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018563.462064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335548928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018563.462073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.462076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018563.462077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018563.462081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.462084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.462088:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018563.462091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018563.462099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335549056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018563.462107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.462110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018563.462111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018563.462115:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.462117:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.462121:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018563.462125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018563.462132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335549184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018563.462142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.462144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018563.462147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018563.462147:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335548672:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018563.462150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.462153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.462156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018563.462161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018563.462168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335549312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018563.462176:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018563.462179:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018563.462180:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018563.462183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2827 00000100:00080000:0.0:1734018563.462186:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018563.462189:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018563.462193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018563.462200:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335549440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018563.462246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462278:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59c9f80 x1818247335548672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335548672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335548672:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018563.462330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018563.462349:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59c8700 x1818247335548800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335548800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018563.462390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018563.462405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59ca300 x1818247335548928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335548928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335548928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018563.462442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018563.462455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59c9180 x1818247335549056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335549056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018563.462522:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335549184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018563.462536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462543:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59c9500 x1818247335549184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335549184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335549184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018563.462573:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335549312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018563.462585:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462591:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59c8000 x1818247335549312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462601:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335549312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335549312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018563.462619:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335549440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018563.462632:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018563.462638:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018563/real 1734018563] req@ffff8800b59cb100 x1818247335549440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018563.462649:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335549440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018563.462657:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018563.462659:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018563.462662:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335549440:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018564.097966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018564.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018565.100045:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018565.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018566.101958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018566.213891:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018567.104030:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018567.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018568.106024:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018568.213883:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018568.469862:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.469871:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018568.469877:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.469889:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.469900:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.469909:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018568.469956:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335549568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018568.469999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.470003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018568.470005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.470010:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.470013:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.470017:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018568.470028:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018568.470040:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335549696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018568.470049:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.470052:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018568.470054:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.470058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.470061:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.470065:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018568.470070:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018568.470078:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335549824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018568.470086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.470089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018568.470091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.470094:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.470097:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.470101:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018568.470106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018568.470111:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335549568:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018568.470115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335549952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018568.470124:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.470127:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018568.470128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.470132:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.470134:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.470138:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018568.470142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018568.470150:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335550080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018568.470158:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.470161:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018568.470163:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.470166:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.470169:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.470172:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018568.470177:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018568.470185:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335550208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018568.470193:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018568.470197:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1734018568.470197:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018568.470199:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018568.470203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2833 00000100:00080000:0.0:1734018568.470206:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018568.470209:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018568.470213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018568.470223:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335550336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018568.470225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59caa00 x1818247335549568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470246:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335549568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470257:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335549568:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018568.470273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018568.470290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018568.470296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59cb480 x1818247335549696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335549696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018568.470329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018568.470344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018568.470350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59cb800 x1818247335549824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335549824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018568.470403:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018568.470417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018568.470423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59c9c00 x1818247335549952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335549952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335549952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018568.470453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018568.470466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018568.470472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59c8a80 x1818247335550080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335550080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018568.470501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018568.470513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018568.470519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59c8e00 x1818247335550208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470529:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335550208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018568.470546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018568.470558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018568.470564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018568/real 1734018568] req@ffff8800b59cbb80 x1818247335550336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018568.470575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335550336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018568.470583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018568.470585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018568.470588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550336:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018569.107984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018569.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018570.109987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018570.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018571.111999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018571.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018572.113910:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018572.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018573.115926:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018573.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018573.477858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.477867:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018573.477872:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.477882:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.477892:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.477899:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018573.477936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335550464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018573.477971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.477975:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018573.477977:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.477981:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.477985:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.477988:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018573.477998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018573.478008:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335550592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018573.478019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.478022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018573.478024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.478027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.478030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.478033:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018573.478038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018573.478047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335550720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018573.478057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.478060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018573.478061:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.478065:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.478067:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.478071:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018573.478075:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018573.478083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335550848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018573.478093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.478096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018573.478097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.478101:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.478104:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.478108:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018573.478112:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018573.478114:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335550464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018573.478120:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335550976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018573.478128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.478131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018573.478133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.478136:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.478138:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.478141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018573.478146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018573.478153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335551104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018573.478161:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018573.478164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018573.478165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018573.478168:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2838 00000100:00080000:0.0:1734018573.478171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018573.478174:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018573.478177:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018573.478185:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335551232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018573.478211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478241:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59ca680 x1818247335550464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335550464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335550464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018573.478294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335550592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018573.478313:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478320:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59cad80 x1818247335550592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335550592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478351:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335550592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018573.478361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335550720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018573.478376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59cbb80 x1818247335550720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335550720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335550720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018573.478439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335550848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018573.478453:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59c8e00 x1818247335550848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335550848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335550848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018573.478490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018573.478504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59c8a80 x1818247335550976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335550976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335550976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018573.478539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335551104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018573.478552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59c9c00 x1818247335551104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478568:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335551104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335551104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018573.478586:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335551232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018573.478599:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018573.478605:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018573/real 1734018573] req@ffff8800b59cb800 x1818247335551232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018573.478616:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335551232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018573.478624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018573.478627:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018573.478630:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335551232:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018574.117972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018574.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018575.119973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018575.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018576.121960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018576.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018577.123959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018577.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018578.125971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018578.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018578.485834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.485843:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018578.485847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018578.485857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.485866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.485874:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018578.485912:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335551360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018578.485947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.485951:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018578.485953:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018578.485957:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.485960:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.485964:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018578.485974:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018578.485985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335551488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018578.485995:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.485998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018578.486000:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018578.486004:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.486006:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.486009:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018578.486015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018578.486023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335551616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018578.486032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.486035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018578.486036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018578.486040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.486043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.486046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018578.486050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018578.486058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335551744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018578.486066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.486069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018578.486071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018578.486074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.486076:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.486080:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018578.486083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018578.486091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335551872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018578.486100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.486103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018578.486105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018578.486106:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335551360:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018578.486109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.486112:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.486116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018578.486120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018578.486128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335552000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018578.486136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018578.486138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018578.486140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018578.486143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2843 00000100:00080000:0.0:1734018578.486145:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018578.486148:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018578.486152:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018578.486161:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335552128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018578.486237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335551488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018578.486259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335551616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018578.486277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335551744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018578.486306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335551872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018578.486323:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018578.486339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552128:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018579.127922:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018579.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018580.129972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018580.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018581.131973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018581.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734018581.492033:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018581.492091:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018581.492116:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018581.492161:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018581.492172:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018581.492180:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018581.492188:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018581.492196:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018581.492202:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018581.492292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59c9500 x1818247335551872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335551872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335551872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018581.492367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59c9180 x1818247335552000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335552000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018581.492401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59ca300 x1818247335552128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335552128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492420:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492423:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492426:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018581.492433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59cb480 x1818247335551360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335551360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492467:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335551360:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018581.492473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59caa00 x1818247335551488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335551488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335551488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018581.492505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59cb100 x1818247335551616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335551616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335551616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018581.492534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018578/real 1734018581] req@ffff8800b59c8000 x1818247335551744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018581.492545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335551744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018581.492559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018581.492609:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018581.492617:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335551744:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018582.133955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018582.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018583.136004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018583.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018583.493860:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.493870:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018583.493874:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.493884:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.493892:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.493900:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018583.493935:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335552256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018583.493978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.493982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018583.493984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.493988:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.493992:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.493995:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018583.494004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018583.494015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335552384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018583.494024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.494027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018583.494029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.494032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.494035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.494038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018583.494042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018583.494052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335552512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018583.494060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.494064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018583.494065:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.494069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.494071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.494075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018583.494080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018583.494088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335552640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018583.494096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.494099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018583.494100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.494104:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.494106:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.494110:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018583.494114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018583.494121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335552768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018583.494125:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335552256:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018583.494131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.494134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018583.494135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.494138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.494141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.494144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018583.494148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018583.494156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335552896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018583.494164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018583.494167:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018583.494169:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018583.494172:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2848 00000100:00080000:0.0:1734018583.494175:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018583.494178:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018583.494182:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018583.494189:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335553024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018583.494220:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59c8700 x1818247335552256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335552256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494291:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494296:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335552256:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018583.494304:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335552384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018583.494323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494330:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59c9f80 x1818247335552384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335552384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335552384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018583.494364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335552512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018583.494379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59c8000 x1818247335552512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335552512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335552512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018583.494416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335552640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018583.494429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59cb100 x1818247335552640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335552640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335552640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018583.494493:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018583.494507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59caa00 x1818247335552768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335552768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494572:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018583.494585:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018583.494603:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494611:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59cb480 x1818247335552896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494622:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335552896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494631:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494633:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335552896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018583.494642:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018583.494655:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018583.494661:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018583/real 1734018583] req@ffff8800b59ca300 x1818247335553024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018583.494671:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335553024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018583.494686:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018583.494688:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018583.494823:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553024:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018584.137930:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018584.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018585.139980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018585.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018586.141950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018586.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018587.143916:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018587.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018588.145977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018588.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018588.501808:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.501817:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018588.501822:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.501831:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.501841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.501848:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018588.501884:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335553152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018588.501918:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.501922:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018588.501924:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.501929:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.501932:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.501935:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018588.501944:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018588.501955:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335553280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018588.501966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.501969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018588.501971:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.501974:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.501976:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.501980:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018588.501984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018588.501992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335553408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018588.502000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.502003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018588.502004:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.502008:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.502010:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.502014:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018588.502018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018588.502026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335553536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018588.502032:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335553152:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018588.502035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.502038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018588.502040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.502043:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.502046:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.502050:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018588.502054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018588.502061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335553664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018588.502069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.502071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018588.502073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.502076:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.502078:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.502081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018588.502086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018588.502092:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335553792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018588.502100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018588.502103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018588.502104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018588.502107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2853 00000100:00080000:0.0:1734018588.502110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018588.502113:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018588.502117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018588.502126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335553920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018588.502130:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502162:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59c9180 x1818247335553152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502184:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335553152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502198:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502201:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335553152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018588.502215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335553280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018588.502233:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59c9500 x1818247335553280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335553280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335553280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018588.502273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335553408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018588.502288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502294:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59cb800 x1818247335553408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335553408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335553408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018588.502349:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335553536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018588.502362:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59c9c00 x1818247335553536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335553536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335553536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018588.502399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018588.502412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59c8a80 x1818247335553664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335553664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018588.502495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018588.502511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59c8e00 x1818247335553792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502529:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335553792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018588.502549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018588.502561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018588.502567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018588/real 1734018588] req@ffff8800b59cbb80 x1818247335553920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018588.502577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335553920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018588.502592:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018588.502594:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018588.502598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335553920:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018589.147961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018589.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018590.149967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018590.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018591.151940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018591.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018592.154002:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018592.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018593.155951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018593.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018593.509809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.509818:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018593.509822:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.509832:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.509841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.509849:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018593.509884:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335554048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018593.509918:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.509923:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018593.509925:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.509929:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.509932:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.509936:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018593.509946:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018593.509957:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335554176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018593.509966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.509970:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018593.509971:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.509975:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.509978:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.509981:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018593.509987:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018593.509997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335554304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018593.510006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.510009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018593.510010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.510014:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.510016:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.510020:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018593.510024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018593.510031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335554432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018593.510041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.510044:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018593.510046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.510049:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.510052:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.510055:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335554048:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018593.510059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018593.510067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335554560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018593.510075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.510077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018593.510080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.510083:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.510086:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.510089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018593.510094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018593.510101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335554688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018593.510109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018593.510111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018593.510113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018593.510116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2858 00000100:00080000:0.0:1734018593.510118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018593.510121:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018593.510125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018593.510132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335554816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018593.510152:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510183:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59cad80 x1818247335554048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510205:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335554048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510219:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335554048:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018593.510236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335554176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018593.510255:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59ca680 x1818247335554176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335554176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335554176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018593.510296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335554304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018593.510311:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59cbb80 x1818247335554304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335554304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335554304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018593.510423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335554432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018593.510441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59c8e00 x1818247335554432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335554432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335554432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018593.510480:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335554560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018593.510493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59c8a80 x1818247335554560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335554560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510522:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510525:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335554560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018593.510530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335554688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018593.510542:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59c9c00 x1818247335554688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510558:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335554688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335554688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018593.510577:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335554816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018593.510590:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018593.510597:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018593/real 1734018593] req@ffff8800b59cb800 x1818247335554816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018593.510607:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335554816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018593.510622:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018593.510625:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018593.510628:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335554816:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018594.157960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018594.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018595.159999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018595.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018596.161960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018596.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018597.163938:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018597.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018598.165974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018598.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018598.517838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.517846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018598.517851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.517861:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.517870:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.517877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018598.517914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335554944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018598.517956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.517960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018598.517961:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.517966:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.517969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.517973:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018598.517982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018598.517993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335555072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018598.518002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.518005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018598.518007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.518011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.518013:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.518016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018598.518021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018598.518028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335555200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018598.518036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.518039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018598.518041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.518045:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.518047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.518051:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018598.518056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018598.518063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335555328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018598.518071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.518074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018598.518076:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.518079:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.518081:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.518085:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018598.518089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018598.518097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335555456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018598.518100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335554944:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018598.518106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.518109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018598.518110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.518114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.518116:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.518120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018598.518124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018598.518132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335555584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018598.518140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018598.518143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018598.518145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018598.518148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2863 00000100:00080000:0.0:1734018598.518151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018598.518153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018598.518157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018598.518166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335555712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018598.518194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59c9500 x1818247335554944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335554944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518261:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518270:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335554944:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018598.518316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018598.518340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518349:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59c9180 x1818247335555072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335555072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018598.518384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018598.518399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59ca300 x1818247335555200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518417:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335555200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518425:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018598.518437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018598.518450:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518456:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59cb480 x1818247335555328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335555328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018598.518515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335555456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018598.518529:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59caa00 x1818247335555456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518547:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335555456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518555:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518557:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518560:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335555456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018598.518567:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335555584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018598.518579:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518585:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59cb100 x1818247335555584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335555584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518602:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335555584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018598.518614:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335555712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018598.518626:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018598.518632:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018598/real 1734018598] req@ffff8800b59c8000 x1818247335555712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018598.518642:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335555712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018598.518657:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018598.518659:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018598.518663:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335555712:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018599.167953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018599.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018600.169955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018600.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018601.172012:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018601.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018602.173885:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018602.213759:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018603.175964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018603.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018603.525846:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.525858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018603.525863:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.525876:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.525889:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018603.525898:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018603.525942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335555840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018603.525988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.525994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018603.525997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.526003:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.526007:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018603.526012:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018603.526022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018603.526036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335555968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018603.526047:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.526051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018603.526052:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.526056:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.526058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018603.526062:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018603.526066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018603.526076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335556096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018603.526085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.526088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018603.526089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.526093:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.526095:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018603.526098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018603.526103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018603.526111:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335556224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018603.526120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.526124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018603.526125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.526129:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.526132:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018603.526133:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335555840:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018603.526135:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018603.526140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018603.526147:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335556352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018603.526155:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.526158:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018603.526160:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.526163:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.526165:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018603.526169:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018603.526173:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018603.526180:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335556480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018603.526231:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018603.526236:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018603.526237:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018603.526241:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2868 00000100:00080000:0.0:1734018603.526245:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018603.526248:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018603.526252:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018603.526261:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335556608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018603.526263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59c9f80 x1818247335555840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335555840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526334:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335555840:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018603.526348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018603.526369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526376:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59c8700 x1818247335555968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335555968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335555968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018603.526410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018603.526426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59c8000 x1818247335556096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335556096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018603.526490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018603.526503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59cb100 x1818247335556224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335556224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018603.526540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335556352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018603.526553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59caa00 x1818247335556352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335556352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526580:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335556352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018603.526588:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335556480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018603.526601:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526607:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59cb480 x1818247335556480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526616:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335556480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526626:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526629:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335556480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018603.526635:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335556608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018603.526647:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018603.526653:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018603/real 1734018603] req@ffff8800b59ca300 x1818247335556608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018603.526663:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335556608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018603.526677:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018603.526680:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018603.526683:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335556608:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018604.177979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018604.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018605.179939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018605.213942:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018606.181932:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018606.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018607.183989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018607.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018608.185970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018608.213799:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018608.533831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.533839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018608.533844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018608.533854:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.533863:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018608.533870:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018608.533906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335556736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018608.533941:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.533945:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018608.533947:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018608.533951:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.533955:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018608.533958:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018608.533969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018608.533979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335556864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018608.533988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.533991:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018608.533993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018608.533998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.534000:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018608.534003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018608.534008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018608.534016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335556992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018608.534024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.534027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018608.534029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018608.534032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.534035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018608.534038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018608.534042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018608.534049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335557120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018608.534057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.534060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018608.534062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018608.534065:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.534067:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018608.534070:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018608.534074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018608.534081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335557248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018608.534091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.534094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018608.534095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018608.534095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335556736:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018608.534099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.534101:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018608.534237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018608.534269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59c9180 x1818247335556736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335556736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018608.534292:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018608.534297:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734018608.534304:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1734018608.534308:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335557376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734018608.534308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335556736:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018608.534318:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018608.534321:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734018608.534322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556864:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734018608.534323:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018608.534327:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2873 00000100:00080000:0.0:1734018608.534331:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018608.534334:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018608.534338:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018608.534342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018608.534348:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335557504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018608.534350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59c9500 x1818247335556864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335556864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018608.534369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018608.534372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018608.534385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018608.534401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018608.534407:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59cb800 x1818247335556992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335556992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018608.534426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018608.534429:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534432:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335556992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018608.534437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335557120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018608.534451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018608.534457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59c9c00 x1818247335557120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335557120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018608.534475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018608.534477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335557120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018608.534515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335557248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018608.534528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018608.534535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59c8a80 x1818247335557248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335557248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018608.534554:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018608.534557:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335557248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018608.534566:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335557376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018608.534578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018608.534584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59c8e00 x1818247335557376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335557376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018608.534602:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018608.534604:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534607:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335557376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018608.534613:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335557504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018608.534626:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018608.534633:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018608/real 1734018608] req@ffff8800b59cbb80 x1818247335557504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018608.534643:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335557504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018608.534658:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018608.534661:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018608.534664:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335557504:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018609.187957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018609.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018610.190002:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018610.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018611.191947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018611.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018612.193907:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018612.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018613.195971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018613.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018613.541822:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.541831:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018613.541836:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.541847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.541856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.541863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018613.541898:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335557632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018613.541932:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.541937:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018613.541939:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.541943:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.541947:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.541950:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018613.541961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018613.541971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335557760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018613.541981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.541984:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018613.541986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.541989:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.541992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.541995:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018613.542000:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018613.542009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335557888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018613.542044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.542048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018613.542049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.542053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.542056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.542059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018613.542063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018613.542072:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335558016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018613.542082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.542085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018613.542087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.542091:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.542093:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.542097:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335557632:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018613.542101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018613.542108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335558144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018613.542116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.542119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018613.542121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.542124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.542127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.542130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018613.542135:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018613.542142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335558272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018613.542150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018613.542153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018613.542154:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018613.542157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2878 00000100:00080000:0.0:1734018613.542160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018613.542163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018613.542167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018613.542174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335558400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018613.542190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59ca680 x1818247335557632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335557632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335557632:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018613.542273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335557760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018613.542292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59cad80 x1818247335557760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335557760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335557760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018613.542332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335557888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018613.542348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59cbb80 x1818247335557888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335557888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335557888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018613.542412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018613.542426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59c8e00 x1818247335558016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335558016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018613.542462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335558144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018613.542475:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59c8a80 x1818247335558144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335558144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335558144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018613.542511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335558272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018613.542523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59c9c00 x1818247335558272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335558272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335558272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018613.542558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335558400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018613.542571:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018613.542577:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018613/real 1734018613] req@ffff8800b59cb800 x1818247335558400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018613.542587:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335558400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018613.542601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018613.542603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018613.542606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335558400:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734018614.197836:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018614.213760:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018615.199958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018615.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018616.202047:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018616.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018617.204002:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018617.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018618.205995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018618.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734018618.549835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.549845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018618.549850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.549860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.549869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.549877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018618.549915:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335558528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018618.549980:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.549985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018618.549986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.549991:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.549995:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.549999:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018618.550008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018618.550020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335558656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018618.550031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.550034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018618.550036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.550039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.550041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.550045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018618.550049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018618.550057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335558784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018618.550066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.550069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018618.550070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.550074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.550077:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.550080:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018618.550084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018618.550091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335558912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018618.550100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.550104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018618.550105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.550109:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.550111:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.550115:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550117:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335558528:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018618.550119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018618.550126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335559040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018618.550134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.550137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018618.550138:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.550141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.550144:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.550147:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018618.550151:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018618.550159:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335559168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018618.550167:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018618.550170:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018618.550172:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018618.550175:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2883 00000100:00080000:0.0:1734018618.550177:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018618.550180:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018618.550184:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018618.550193:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335559296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018618.550216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550248:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59c9500 x1818247335558528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335558528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335558528:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018618.550300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018618.550318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59c9180 x1818247335558656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335558656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550352:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018618.550359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018618.550375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59ca300 x1818247335558784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335558784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018618.550439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018618.550453:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59cb480 x1818247335558912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335558912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335558912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018618.550490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018618.550503:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59caa00 x1818247335559040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335559040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018618.550539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018618.550552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59cb100 x1818247335559168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550568:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335559168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018618.550586:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018618.550599:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018618.550605:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018618/real 1734018618] req@ffff8800b59c8000 x1818247335559296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018618.550616:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335559296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018618.550631:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018618.550634:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018618.550637:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559296:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734018619.208001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018619.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018620.210001:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018620.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018621.211957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018621.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734018622.213947:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018622.213992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018623.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018623.215930:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018623.557826:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.557835:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018623.557840:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.557889:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.557898:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.557907:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018623.557944:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335559424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018623.557979:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.557983:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018623.557985:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.557990:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.557994:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.557997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018623.558007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018623.558018:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335559552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018623.558028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.558031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018623.558033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.558036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.558038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.558042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018623.558047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018623.558056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335559680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018623.558065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.558068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018623.558070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.558073:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.558075:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.558079:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018623.558083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018623.558091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335559808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018623.558109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.558112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018623.558113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.558117:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.558119:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.558123:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018623.558127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018623.558134:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335559424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018623.558135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335559936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018623.558143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.558146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018623.558147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.558150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.558153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.558156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018623.558160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018623.558167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335560064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018623.558175:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018623.558178:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018623.558179:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018623.558183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2888 00000100:00080000:0.0:1734018623.558186:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018623.558189:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018623.558193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018623.558200:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335560192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018623.558230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558261:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59c8700 x1818247335559424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335559424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335559424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018623.558314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335559552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018623.558332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59c9f80 x1818247335559552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335559552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335559552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018623.558373:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335559680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018623.558388:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558394:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59c8000 x1818247335559680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335559680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335559680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018623.558473:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335559808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018623.558489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59cb100 x1818247335559808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335559808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335559808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018623.558535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018623.558549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59caa00 x1818247335559936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335559936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335559936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018623.558586:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018623.558598:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59cb480 x1818247335560064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558614:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335560064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558624:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558627:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018623.558632:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018623.558645:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018623.558652:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018623/real 1734018623] req@ffff8800b59ca300 x1818247335560192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018623.558662:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335560192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018623.558677:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018623.558679:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018623.558683:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018624.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018624.217930:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018625.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018625.220051:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018626.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018626.221976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018627.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018627.223948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018628.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018628.226024:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018628.565831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.565840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018628.565844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.565854:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.565863:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.565871:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018628.565905:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335560320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018628.565938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.565943:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018628.565945:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.565949:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.565952:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.565956:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018628.565965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018628.565976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335560448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018628.565985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.565988:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018628.565991:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.565995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.565997:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.566001:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018628.566005:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018628.566013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335560576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018628.566021:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.566025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018628.566026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.566030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.566032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.566035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018628.566039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018628.566046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335560704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018628.566056:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.566059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018628.566060:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.566064:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.566066:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.566070:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018628.566074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018628.566076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335560320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018628.566082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335560832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018628.566091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.566094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018628.566096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.566099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.566102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.566105:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018628.566109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018628.566116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335560960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018628.566123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018628.566126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018628.566128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018628.566131:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2893 00000100:00080000:0.0:1734018628.566133:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018628.566136:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018628.566140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018628.566149:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335561088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018628.566172:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59c9180 x1818247335560320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335560320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335560320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018628.566256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335560448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018628.566275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59c9500 x1818247335560448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335560448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335560448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018628.566315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335560576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018628.566331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59cb800 x1818247335560576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566347:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335560576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335560576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018628.566391:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335560704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018628.566404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566410:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59c9c00 x1818247335560704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335560704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335560704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018628.566440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018628.566453:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59c8a80 x1818247335560832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335560832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018628.566490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018628.566501:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59c8e00 x1818247335560960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335560960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335560960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018628.566535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018628.566548:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018628.566554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018628/real 1734018628] req@ffff8800b59cbb80 x1818247335561088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018628.566564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335561088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018628.566578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018628.566580:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018628.566583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018629.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018629.227977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018630.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018630.229971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018631.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018631.231934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018632.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018632.233963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018633.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018633.235955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018633.573832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.573841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018633.573846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.573856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.573864:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018633.573872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018633.573909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335561216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018633.573944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.573948:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018633.573950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.573955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.573958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018633.573961:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018633.573971:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018633.573982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335561344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018633.573992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.573995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018633.573996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.574000:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.574003:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018633.574006:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018633.574010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018633.574019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335561472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018633.574028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.574031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018633.574034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.574037:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.574040:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018633.574043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018633.574047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018633.574055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335561600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018633.574065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.574068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018633.574069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.574073:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.574076:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018633.574078:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335561216:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018633.574079:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018633.574083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018633.574091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335561728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018633.574099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.574102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018633.574103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.574107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.574109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018633.574112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018633.574116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018633.574124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335561856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018633.574132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018633.574135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018633.574137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018633.574140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2898 00000100:00080000:0.0:1734018633.574143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018633.574146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018633.574150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018633.574157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335561984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018633.574170:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59cad80 x1818247335561216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335561216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574236:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574239:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335561216:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018633.574253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335561344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018633.574271:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574278:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59ca680 x1818247335561344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335561344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335561344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018633.574312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335561472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018633.574327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59cbb80 x1818247335561472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335561472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335561472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018633.574390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335561600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018633.574404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574410:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59c8e00 x1818247335561600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335561600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335561600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018633.574440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018633.574452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574459:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59c8a80 x1818247335561728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335561728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018633.574489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018633.574501:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59c9c00 x1818247335561856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335561856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018633.574535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018633.574547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018633.574553:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018633/real 1734018633] req@ffff8800b59cb800 x1818247335561984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018633.574563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335561984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018633.574578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018633.574581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018633.574584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335561984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018634.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018634.237997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018635.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018635.239939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018636.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018636.241907:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018637.213797:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018637.244002:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018638.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018638.245970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018638.581791:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.581800:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018638.581805:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.581815:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00080000:0.0:1734018638.581824:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.581832:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018638.581870:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335562112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018638.581905:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.581909:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018638.581911:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.581916:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00080000:0.0:1734018638.581919:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.581922:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018638.581934:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018638.581944:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335562240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018638.581955:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.581958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018638.581960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.581963:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00080000:0.0:1734018638.581966:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.581969:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018638.581974:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018638.581982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335562368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018638.581990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.581994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018638.581995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.581998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00080000:0.0:1734018638.582001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.582004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018638.582008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018638.582016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335562496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018638.582025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.582028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018638.582031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.582034:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00100000:3.0:1734018638.582036:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335562112:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018638.582037:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.582040:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018638.582044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018638.582052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335562624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018638.582060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.582063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018638.582064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.582068:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00080000:0.0:1734018638.582070:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.582073:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018638.582077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018638.582084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335562752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018638.582092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018638.582095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018638.582097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018638.582100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2903 00000100:00080000:0.0:1734018638.582102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018638.582105:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018638.582109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018638.582119:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335562880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018638.582135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582167:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59c9500 x1818247335562112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582189:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335562112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582202:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582211:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335562112:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018638.582219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335562240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018638.582238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59c9180 x1818247335562240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582257:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335562240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335562240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018638.582279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335562368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018638.582295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582301:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59ca300 x1818247335562368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335562368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335562368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018638.582358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335562496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018638.582371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59cb480 x1818247335562496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335562496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335562496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018638.582408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335562624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018638.582421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59caa00 x1818247335562624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335562624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335562624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018638.582457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335562752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018638.582469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59cb100 x1818247335562752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335562752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335562752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018638.582503:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335562880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018638.582517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018638.582523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018638/real 1734018638] req@ffff8800b59c8000 x1818247335562880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018638.582533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335562880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018638.582548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018638.582550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018638.582553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335562880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018639.213779:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018639.247941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018640.213939:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018640.250000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018641.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018641.251976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018642.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018642.253941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018643.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018643.255950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018643.589812:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.589823:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018643.589829:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.589842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.589853:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.589863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018643.589910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335563008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018643.589949:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.589953:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018643.589955:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.589960:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.589964:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.589967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018643.589979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018643.589991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335563136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018643.590001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.590004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018643.590006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.590011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.590014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.590017:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018643.590022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018643.590034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335563264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018643.590042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.590045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018643.590047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.590050:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.590053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.590056:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018643.590060:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018643.590068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335563008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018643.590069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335563392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018643.590078:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.590081:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018643.590083:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.590087:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.590089:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.590093:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018643.590098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018643.590105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335563520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018643.590115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.590118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018643.590120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.590123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.590126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.590130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018643.590134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018643.590141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335563648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018643.590149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018643.590152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018643.590153:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018643.590157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2908 00000100:00080000:0.0:1734018643.590160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018643.590163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018643.590167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018643.590175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335563776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018643.590184:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335563136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018643.590207:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335563264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018643.590234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335563392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018643.590251:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335563520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018643.590267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335563648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018643.590282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335563776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018644.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018644.257973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018645.213948:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018645.259961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018646.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018646.261989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018646.596008:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018646.596063:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018646.596087:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018646.596129:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018646.596137:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018646.596146:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018646.596153:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018646.596161:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018646.596168:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018646.596236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59cb100 x1818247335563392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335563392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596284:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335563392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018646.596304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59caa00 x1818247335563520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335563520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596325:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335563520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018646.596338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59cb480 x1818247335563648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335563648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335563648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018646.596370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59ca300 x1818247335563776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335563776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596394:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335563776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018646.596403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59c9f80 x1818247335563008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335563008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335563008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018646.596448:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59c8700 x1818247335563136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335563136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335563136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018646.596524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018643/real 1734018646] req@ffff8800b59c8000 x1818247335563264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018646.596535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335563264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018646.596543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018646.596545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018646.596548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335563264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018647.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018647.263885:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018648.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018648.265897:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018648.597839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018648.597851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018648.597857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.597871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.597883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.597892:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018648.597941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335563904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018648.597983:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018648.597987:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018648.597989:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.597994:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.597998:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.598002:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018648.598013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018648.598025:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335564032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018648.598035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018648.598039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018648.598040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.598044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.598047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.598052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018648.598056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018648.598065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335564160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018648.598073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018648.598079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018648.598081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.598085:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.598088:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.598091:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018648.598095:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018648.598103:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335564288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018648.598112:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734018648.598112:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335563904:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018648.598115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018648.598117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.598121:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.598123:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.598127:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018648.598131:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018648.598139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335564416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018648.598147:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018648.598150:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018648.598151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.598155:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.598157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.598161:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018648.598166:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018648.598174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335564544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018648.598182:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018648.598185:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018648.598186:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018648.598190:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2913 00000100:00080000:0.0:1734018648.598192:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018648.598197:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734018648.598199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734018648.598201:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018648.598211:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335564672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018648.598226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59c9180 x1818247335563904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598248:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335563904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335563904:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018648.598276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018648.598293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018648.598300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59c9500 x1818247335564032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335564032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018648.598332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018648.598346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018648.598353:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59cb800 x1818247335564160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335564160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018648.598407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018648.598420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018648.598478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59c9c00 x1818247335564288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598490:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335564288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018648.598509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335564416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018648.598526:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018648.598533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59c8a80 x1818247335564416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335564416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598551:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335564416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018648.598563:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335564544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018648.598576:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018648.598582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59c8e00 x1818247335564544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598593:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335564544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335564544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018648.598611:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335564672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018648.598624:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018648.598630:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018648/real 1734018648] req@ffff8800b59cbb80 x1818247335564672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018648.598640:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335564672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018648.598647:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018648.598649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018648.598652:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335564672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018649.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018649.267972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018650.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018650.269942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018651.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018651.271956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018652.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018652.273967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018653.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018653.275970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018653.605852:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.605861:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018653.605865:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.605875:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.605884:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.605892:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018653.605927:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335564800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018653.605964:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.605968:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018653.605969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.605974:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.605978:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.605982:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018653.605992:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018653.606003:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335564928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018653.606013:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.606016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018653.606017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.606021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.606023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.606026:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018653.606031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018653.606041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335565056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018653.606050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.606053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018653.606054:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.606058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.606060:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.606063:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018653.606067:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018653.606075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335565184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018653.606091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.606094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018653.606095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.606099:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.606102:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.606105:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606105:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335564800:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018653.606109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018653.606117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335565312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018653.606124:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.606127:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018653.606129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.606132:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.606134:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.606138:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018653.606142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018653.606149:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335565440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018653.606157:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018653.606160:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018653.606161:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018653.606164:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2918 00000100:00080000:0.0:1734018653.606167:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018653.606170:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018653.606174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018653.606182:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335565568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018653.606199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606229:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59ca680 x1818247335564800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335564800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606264:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335564800:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018653.606282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018653.606300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59cad80 x1818247335564928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335564928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335564928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018653.606385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018653.606404:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59cbb80 x1818247335565056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335565056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018653.606478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018653.606492:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606498:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59c8e00 x1818247335565184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606509:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335565184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018653.606529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335565312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018653.606541:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59c8a80 x1818247335565312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335565312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335565312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018653.606577:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335565440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018653.606590:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606596:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59c9c00 x1818247335565440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606606:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335565440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606616:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606619:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335565440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018653.606624:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335565568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018653.606637:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018653.606643:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018653/real 1734018653] req@ffff8800b59cb800 x1818247335565568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018653.606653:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335565568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018653.606660:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018653.606662:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018653.606665:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335565568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018654.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018654.277952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018655.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018655.280001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018656.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018656.281947:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018657.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018657.283951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018658.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018658.285974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018658.613829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.613838:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018658.613842:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.613852:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.613861:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.613869:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018658.613905:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335565696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018658.613939:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.613943:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018658.613945:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.613949:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.613952:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.613956:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018658.613965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018658.613976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335565824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018658.613985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.613989:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018658.613992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.613995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.613998:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.614001:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018658.614006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018658.614014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335565952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018658.614022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.614025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018658.614027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.614030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.614032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.614036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018658.614040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018658.614047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335566080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018658.614056:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.614059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018658.614061:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.614064:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.614067:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.614070:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614073:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335565696:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018658.614074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018658.614082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335566208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018658.614091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.614094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018658.614095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.614099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.614101:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.614104:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018658.614108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018658.614115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335566336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018658.614123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018658.614126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018658.614127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018658.614130:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2923 00000100:00080000:0.0:1734018658.614132:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018658.614135:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018658.614139:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018658.614148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335566464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018658.614167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59c9500 x1818247335565696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335565696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335565696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018658.614250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018658.614305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59c9180 x1818247335565824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335565824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614341:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018658.614347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018658.614363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59c8000 x1818247335565952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335565952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335565952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018658.614435:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018658.614448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59c8700 x1818247335566080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335566080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018658.614485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335566208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018658.614499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59c9f80 x1818247335566208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335566208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335566208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018658.614536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335566336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018658.614549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614555:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59ca300 x1818247335566336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614565:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335566336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614573:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335566336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018658.614583:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335566464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018658.614596:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018658.614602:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018658/real 1734018658] req@ffff8800b59cb480 x1818247335566464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018658.614611:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335566464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018658.614619:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018658.614621:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018658.614625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335566464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018659.213936:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018659.287971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018660.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018660.289953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018661.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018661.291986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018662.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018662.293933:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018663.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018663.295977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018663.621837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.621848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018663.621854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.621868:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.621878:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018663.621888:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018663.621937:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335566592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018663.621977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.621982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018663.621984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.621989:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.621993:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018663.621997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018663.622008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018663.622019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335566720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018663.622029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.622033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018663.622034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.622038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.622041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018663.622045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018663.622050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018663.622060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335566848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018663.622069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.622073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018663.622075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.622079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.622081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018663.622084:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018663.622088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018663.622097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335566976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018663.622102:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335566592:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018663.622106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.622109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018663.622111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.622115:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.622117:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018663.622120:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018663.622125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018663.622132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335567104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018663.622140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.622143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018663.622145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.622149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.622152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018663.622156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018663.622160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018663.622209:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335567232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018663.622218:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018663.622221:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018663.622223:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018663.622227:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2928 00000100:00080000:0.0:1734018663.622230:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018663.622230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018663.622233:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018663.622238:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018663.622248:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335567360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018663.622264:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59caa00 x1818247335566592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335566592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335566592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018663.622315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018663.622339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018663.622347:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59cb100 x1818247335566720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335566720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622366:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018663.622378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018663.622392:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018663.622398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59cb480 x1818247335566848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335566848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018663.622461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018663.622474:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018663.622480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59ca300 x1818247335566976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335566976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335566976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018663.622509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335567104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018663.622521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018663.622527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59c9f80 x1818247335567104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335567104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335567104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018663.622556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335567232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018663.622568:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018663.622574:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59c8700 x1818247335567232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335567232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335567232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018663.622601:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335567360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018663.622613:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018663.622619:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018663/real 1734018663] req@ffff8800b59c8000 x1818247335567360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018663.622628:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335567360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018663.622635:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018663.622637:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018663.622640:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335567360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018664.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018664.297973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018665.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018665.299933:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018666.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018666.301950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018667.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018667.303976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018668.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018668.305959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018668.629832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.629841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018668.629845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018668.629855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.629863:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.629871:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018668.629910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335567488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018668.629944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.629948:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018668.629950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018668.629955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.629958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.629961:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018668.629971:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018668.629981:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335567616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018668.629991:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.629994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018668.629995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018668.629999:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.630001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.630004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018668.630009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018668.630017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335567744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018668.630025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.630028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018668.630029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018668.630033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.630036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.630040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018668.630044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018668.630051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335567872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018668.630059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.630062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018668.630064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018668.630067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.630070:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.630073:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018668.630076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018668.630125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335568000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018668.630136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.630139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018668.630140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018668.630140:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335567488:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018668.630144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.630147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.630151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018668.630155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018668.630164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335568128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018668.630173:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018668.630176:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018668.630178:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018668.630181:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2933 00000100:00080000:0.0:1734018668.630183:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018668.630186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018668.630190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018668.630199:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335568256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018668.630239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59c9180 x1818247335567488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335567488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335567488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018668.630326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335567616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018668.630345:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59c9500 x1818247335567616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335567616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335567616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018668.630384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335567744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018668.630399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59cb800 x1818247335567744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335567744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335567744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018668.630443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335567872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018668.630456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59c9c00 x1818247335567872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335567872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335567872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018668.630518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018668.630532:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630538:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59c8a80 x1818247335568000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335568000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630560:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018668.630569:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018668.630582:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630588:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59c8e00 x1818247335568128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630598:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335568128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630606:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630608:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630611:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018668.630617:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018668.630629:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018668.630635:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018668/real 1734018668] req@ffff8800b59cbb80 x1818247335568256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018668.630645:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335568256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018668.630653:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018668.630656:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018668.630659:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018669.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018669.307950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018670.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018670.309996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018671.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018671.311945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018672.213917:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018672.313877:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018673.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018673.315955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018673.637837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.637846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018673.637851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018673.637862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.637871:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.637879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018673.637914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335568384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018673.637948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.637953:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018673.637955:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018673.637959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.637963:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.637967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018673.637976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018673.637987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335568512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018673.638038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.638042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018673.638044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018673.638048:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.638051:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.638055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018673.638060:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018673.638071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335568640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018673.638083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.638086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018673.638088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018673.638092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.638094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.638098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018673.638102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018673.638106:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335568384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018673.638110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335568768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018673.638126:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.638129:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018673.638130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018673.638134:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.638136:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.638140:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018673.638144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018673.638151:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335568896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018673.638159:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.638162:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018673.638163:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018673.638167:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.638171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.638174:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018673.638177:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018673.638184:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335569024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018673.638192:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018673.638195:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018673.638197:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734018673.638198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018673.638200:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2938 00000100:00080000:0.0:1734018673.638204:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018673.638207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018673.638211:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018673.638219:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335569152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018673.638229:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59cad80 x1818247335568384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638251:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335568384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638264:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335568384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018673.638282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335568512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018673.638300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018673.638308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59ca680 x1818247335568512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335568512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335568512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018673.638364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335568640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018673.638379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018673.638386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59cbb80 x1818247335568640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335568640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638418:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335568640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018673.638424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335568768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018673.638437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018673.638443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59c8e00 x1818247335568768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335568768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335568768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018673.638472:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018673.638486:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018673.638492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59c8a80 x1818247335568896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335568896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638509:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638511:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335568896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018673.638521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018673.638533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018673.638539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59c9c00 x1818247335569024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335569024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018673.638567:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018673.638579:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018673.638585:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018673/real 1734018673] req@ffff8800b59cb800 x1818247335569152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018673.638595:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335569152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018673.638602:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018673.638604:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018673.638607:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018674.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018674.317942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018675.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018675.319960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018676.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018676.322041:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018677.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018677.323972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018678.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018678.325986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018678.645810:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.645819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018678.645824:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.645835:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.645843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018678.645851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018678.645887:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335569280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018678.645951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.645955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018678.645958:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.645962:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.645966:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018678.645969:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018678.645980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018678.645992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335569408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018678.646002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.646005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018678.646008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.646012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.646015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018678.646018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018678.646022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018678.646030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335569536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018678.646040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.646043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018678.646044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.646048:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.646050:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018678.646054:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646054:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335569280:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018678.646058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018678.646066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335569664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018678.646074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.646077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018678.646078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.646081:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.646084:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018678.646087:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018678.646092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018678.646099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335569792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018678.646108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.646111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018678.646112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.646116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.646118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018678.646121:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018678.646125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018678.646132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335569920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018678.646139:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018678.646142:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018678.646144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018678.646147:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2943 00000100:00080000:0.0:1734018678.646149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018678.646150:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018678.646153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018678.646157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018678.646165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335570048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018678.646181:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59c9500 x1818247335569280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646204:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335569280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646218:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646222:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646227:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335569280:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018678.646235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335569408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018678.646255:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59c9180 x1818247335569408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335569408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335569408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018678.646317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335569536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018678.646333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59c8000 x1818247335569536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335569536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646365:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335569536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018678.646378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335569664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018678.646390:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646397:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59c8700 x1818247335569664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335569664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335569664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018678.646427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018678.646440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59c9f80 x1818247335569792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335569792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018678.646476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018678.646489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59ca300 x1818247335569920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335569920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335569920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018678.646524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018678.646536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018678.646542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018678/real 1734018678] req@ffff8800b59cb480 x1818247335570048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018678.646552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335570048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018678.646559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018678.646562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018678.646565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018679.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018679.328002:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018680.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018680.329975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018681.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018681.331956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018682.213923:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018682.333998:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018683.213908:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018683.335954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018683.653857:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.653868:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018683.653873:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018683.653885:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.653896:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.653906:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018683.653954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335570176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018683.653992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.653996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018683.653999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018683.654004:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.654007:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.654011:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018683.654023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018683.654034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335570304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018683.654043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.654047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018683.654049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018683.654053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.654056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.654059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018683.654064:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018683.654075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335570432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018683.654083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.654087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018683.654089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018683.654094:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.654097:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.654100:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018683.654104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018683.654113:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335570560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018683.654114:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335570176:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018683.654122:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.654125:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018683.654127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018683.654131:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.654133:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.654137:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018683.654141:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018683.654149:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335570688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018683.654156:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.654160:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018683.654161:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018683.654165:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.654168:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.654171:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018683.654175:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018683.654183:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335570816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018683.654192:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018683.654195:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018683.654197:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734018683.654198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018683.654201:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2948 00000100:00080000:0.0:1734018683.654203:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018683.654207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018683.654211:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018683.654219:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335570944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018683.654228:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59cb100 x1818247335570176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335570176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335570176:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018683.654276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335570304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018683.654293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018683.654300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59caa00 x1818247335570304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335570304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335570304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018683.654331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335570432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018683.654346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018683.654352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59cb480 x1818247335570432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335570432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335570432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018683.654416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335570560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018683.654428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018683.654434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59ca300 x1818247335570560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335570560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335570560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018683.654464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018683.654476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018683.654481:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59c9f80 x1818247335570688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335570688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018683.654511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018683.654524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018683.654530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59c8700 x1818247335570816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335570816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018683.654557:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018683.654569:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018683.654575:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018683/real 1734018683] req@ffff8800b59c8000 x1818247335570944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018683.654584:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335570944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018683.654591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018683.654593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018683.654597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335570944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018684.213887:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018684.338009:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018685.213887:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018685.340061:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018686.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018686.341980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018687.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018687.343950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018688.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018688.346015:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018688.661847:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.661857:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018688.661862:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.661872:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.661881:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.661889:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018688.661925:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335571072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018688.661960:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.661964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018688.661966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.661971:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.661974:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.661977:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018688.661987:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018688.661998:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335571200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018688.662008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.662012:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018688.662013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.662017:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.662019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.662023:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018688.662027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018688.662035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335571328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018688.662044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.662047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018688.662048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.662052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.662054:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.662057:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018688.662061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018688.662069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335571456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018688.662078:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.662082:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018688.662084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.662088:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.662090:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.662094:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018688.662098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018688.662100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335571072:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018688.662106:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335571584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018688.662114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.662117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018688.662118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.662122:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.662124:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.662127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018688.662132:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018688.662138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335571712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018688.662146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018688.662149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018688.662150:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018688.662153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2953 00000100:00080000:0.0:1734018688.662156:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018688.662159:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018688.662163:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018688.662172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335571840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018688.662195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662227:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59c9180 x1818247335571072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335571072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335571072:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018688.662279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335571200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018688.662297:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59c9500 x1818247335571200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662316:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335571200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662327:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662330:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335571200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018688.662337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335571328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018688.662351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59cb800 x1818247335571328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335571328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335571328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018688.662419:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335571456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018688.662432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59c9c00 x1818247335571456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335571456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335571456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018688.662469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335571584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018688.662482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59c8a80 x1818247335571584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335571584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335571584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018688.662517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335571712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018688.662530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59c8e00 x1818247335571712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335571712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335571712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018688.662564:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335571840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018688.662577:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018688.662583:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018688/real 1734018688] req@ffff8800b59cbb80 x1818247335571840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018688.662592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335571840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018688.662600:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018688.662602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018688.662605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335571840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018689.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018689.347950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018690.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018690.349954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018691.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018691.351998:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018692.213750:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018692.353944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018693.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018693.355939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018693.669824:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.669833:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018693.669838:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.669848:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.669857:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.669865:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018693.669901:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335571968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018693.669937:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.669941:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018693.669943:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.669947:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.669950:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.669954:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018693.669964:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018693.669975:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335572096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018693.669984:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.669988:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018693.669989:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.669993:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.669996:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.669999:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018693.670004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018693.670014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335572224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018693.670023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.670026:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018693.670028:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.670031:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.670034:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.670037:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018693.670041:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018693.670048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335572352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018693.670058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.670061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018693.670062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.670066:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.670068:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.670072:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018693.670076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018693.670082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335571968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018693.670084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335572480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018693.670092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.670095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018693.670097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.670101:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.670103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.670106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018693.670111:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018693.670118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335572608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018693.670126:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018693.670128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018693.670130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018693.670133:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2958 00000100:00080000:0.0:1734018693.670135:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018693.670138:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018693.670142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018693.670149:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335572736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018693.670178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59ca680 x1818247335571968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335571968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670255:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335571968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018693.670263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018693.670282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59cad80 x1818247335572096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335572096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018693.670322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018693.670337:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59cbb80 x1818247335572224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335572224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018693.670407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018693.670420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59c8e00 x1818247335572352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335572352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018693.670457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335572480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018693.670470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59c8a80 x1818247335572480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335572480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335572480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018693.670505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335572608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018693.670517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59c9c00 x1818247335572608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335572608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670541:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670544:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670546:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335572608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018693.670552:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335572736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018693.670564:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018693.670570:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018693/real 1734018693] req@ffff8800b59cb800 x1818247335572736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018693.670580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335572736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018693.670587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018693.670589:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018693.670593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335572736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018694.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018694.357952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018695.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018695.359954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018696.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018696.361956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018697.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018697.364004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018698.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018698.365994:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018698.677849:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018698.677861:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018698.677866:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018698.677881:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.677892:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.677902:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018698.677952:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335572864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018698.678000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018698.678007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018698.678010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018698.678017:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.678023:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.678029:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018698.678043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018698.678064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335572992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018698.678070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335572864:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018698.678080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018698.678084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018698.678087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018698.678092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.678096:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.678102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018698.678109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018698.678121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335573120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018698.678133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018698.678137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018698.678140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018698.678145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.678149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.678155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018698.678162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018698.678162:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018698.678174:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335573248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018698.678186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018698.678191:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018698.678194:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018698.678194:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59c9500 x1818247335572864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018698.678200:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.678205:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.678210:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018698.678215:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018698.678220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335572864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018698.678228:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335573376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734018698.678237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734018698.678241:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1734018698.678242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734018698.678247:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1734018698.678248:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335572864:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018698.678249:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018698.678255:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.678259:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.678264:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018698.678269:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018698.678281:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335573504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018698.678283:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572992:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734018698.678295:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018698.678301:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018698.678304:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734018698.678309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018698.678310:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2963 00000100:00080000:0.0:1734018698.678315:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018698.678320:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018698.678321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59c9180 x1818247335572992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018698.678327:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018698.678338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335572992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018698.678341:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335573632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734018698.678352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018698.678356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018698.678361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335572992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018698.678370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335573120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018698.678397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018698.678409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59c8000 x1818247335573120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018698.678428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335573120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018698.678447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018698.678450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018698.678453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335573120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018698.678460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335573248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018698.678479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018698.678486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59c8700 x1818247335573248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018698.678497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335573248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018698.678505:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018698.678507:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018698.678510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335573248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018698.678515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335573376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018698.678528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018698.678535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59c9f80 x1818247335573376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018698.678544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335573376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018698.678552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018698.678554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018698.678558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335573376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018698.678630:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335573504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018698.678648:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018698.678657:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59ca300 x1818247335573504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018698.678668:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335573504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018698.678677:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018698.678679:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018698.678682:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335573504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018698.678687:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335573632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018698.678772:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018698.678780:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018698/real 1734018698] req@ffff8800b59cb480 x1818247335573632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018698.678792:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335573632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018698.678800:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018698.678802:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018698.678805:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335573632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018699.213799:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018699.367950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018700.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018700.369978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018701.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018701.371943:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018702.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018702.373987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018703.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018703.376015:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018703.685789:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.685799:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018703.685803:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.685814:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.685822:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.685830:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018703.685866:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335573760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018703.685902:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.685906:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018703.685908:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.685913:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.685916:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.685919:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018703.685929:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018703.685939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335573888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018703.685950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.685953:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018703.685955:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.685958:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.685961:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.685964:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018703.685968:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018703.685978:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335574016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018703.685986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.685990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018703.685991:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.685995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.685997:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.686000:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018703.686004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018703.686012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335574144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018703.686025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.686028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018703.686029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.686033:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.686035:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.686039:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018703.686044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018703.686048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335573760:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018703.686052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335574272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018703.686060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.686063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018703.686064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.686067:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.686070:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.686073:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018703.686077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018703.686084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335574400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018703.686092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018703.686094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018703.686096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018703.686099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2968 00000100:00080000:0.0:1734018703.686101:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018703.686104:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018703.686108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018703.686115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335574528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018703.686144:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686176:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59caa00 x1818247335573760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686198:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335573760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686212:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335573760:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018703.686229:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335573888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018703.686248:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59cb100 x1818247335573888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686266:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335573888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686277:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335573888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018703.686286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018703.686301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59cb480 x1818247335574016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335574016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686338:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018703.686372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018703.686386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59ca300 x1818247335574144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335574144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018703.686423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335574272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018703.686435:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59c9f80 x1818247335574272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335574272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335574272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018703.686471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335574400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018703.686483:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59c8700 x1818247335574400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335574400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335574400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018703.686561:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335574528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018703.686579:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018703.686586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018703/real 1734018703] req@ffff8800b59c8000 x1818247335574528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018703.686596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335574528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018703.686604:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018703.686607:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018703.686610:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335574528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018704.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018704.377951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018705.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018705.379958:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018706.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018706.382020:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018707.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018707.383911:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018708.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018708.385838:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018708.693836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.693846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018708.693851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.693861:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.693871:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.693878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018708.693915:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335574656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018708.693950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.693954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018708.693956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.693960:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.693963:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.693967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018708.693976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018708.693986:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335574784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018708.693996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.693999:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018708.694001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.694005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.694007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.694011:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018708.694016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018708.694024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335574912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018708.694033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.694035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018708.694037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.694040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.694043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.694047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018708.694051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018708.694058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335575040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018708.694066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.694069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018708.694070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.694074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.694077:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.694080:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018708.694084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018708.694091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335575168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018708.694100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.694103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018708.694105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.694109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.694111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.694115:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018708.694119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018708.694122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335574656:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018708.694126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335575296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018708.694134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018708.694137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018708.694138:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018708.694141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2973 00000100:00080000:0.0:1734018708.694144:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018708.694147:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018708.694151:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018708.694157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335575424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018708.694262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018708.694287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018708.694305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018708.694334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335575168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018708.694351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335575296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018708.694367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335575424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018709.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018709.387924:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018710.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018710.389880:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018711.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018711.391771:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018711.700033:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018711.700089:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018711.700114:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018711.700159:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018711.700168:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018711.700176:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018711.700183:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018711.700191:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018711.700200:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018711.700258:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59c8a80 x1818247335575168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700291:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335575168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335575168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018711.700325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59c8e00 x1818247335575296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335575296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335575296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018711.700406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59cbb80 x1818247335575424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335575424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335575424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018711.700440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59c9180 x1818247335574656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335574656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335574656:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018711.700473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59c9500 x1818247335574784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335574784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018711.700505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59cb800 x1818247335574912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335574912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700528:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335574912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018711.700537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018708/real 1734018711] req@ffff8800b59c9c00 x1818247335575040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018711.700548:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335575040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018711.700555:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018711.700558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018711.700561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018712.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018712.393891:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018713.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018713.395984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018713.701834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.701844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018713.701848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.701858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.701868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.701876:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018713.701911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335575552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018713.701954:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.701958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018713.701960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.701964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.701968:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.701971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018713.701981:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018713.701992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335575680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018713.702001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.702004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018713.702006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.702009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.702012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.702015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018713.702020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018713.702030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335575808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018713.702038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.702042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018713.702043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.702047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.702049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.702053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018713.702057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018713.702065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335575936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018713.702075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.702078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018713.702079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.702083:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.702085:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.702089:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018713.702093:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018713.702094:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335575552:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018713.702101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335576064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018713.702109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.702112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018713.702113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.702116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.702119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.702122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018713.702126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018713.702134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335576192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018713.702142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018713.702146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018713.702147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018713.702151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2978 00000100:00080000:0.0:1734018713.702153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018713.702156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018713.702160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018713.702167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335576320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018713.702188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59cad80 x1818247335575552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702241:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335575552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335575552:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018713.702280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018713.702298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59ca680 x1818247335575680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335575680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018713.702381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018713.702399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702407:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59c9c00 x1818247335575808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335575808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702429:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702432:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018713.702463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018713.702477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59cb800 x1818247335575936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335575936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335575936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018713.702514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018713.702527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59c9500 x1818247335576064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335576064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018713.702561:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018713.702574:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702580:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59c9180 x1818247335576192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335576192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702600:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018713.702611:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018713.702623:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018713.702629:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018713/real 1734018713] req@ffff8800b59cbb80 x1818247335576320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018713.702639:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335576320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018713.702646:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018713.702649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018713.702652:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018714.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018714.397963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018715.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018715.399966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018716.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018716.401982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018717.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018717.403944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018718.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018718.406306:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018718.709832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.709844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018718.709848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018718.709858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.709867:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.709875:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018718.709911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335576448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018718.709945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.709949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018718.709951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018718.709956:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.709959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.709963:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018718.709972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018718.709983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335576576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018718.709993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.709997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018718.709998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018718.710002:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.710004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.710007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018718.710012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018718.710019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335576704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018718.710028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.710031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018718.710033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018718.710036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.710039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.710042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018718.710046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018718.710054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335576832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018718.710063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.710066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018718.710068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018718.710070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335576448:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018718.710072:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.710075:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.710079:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018718.710083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018718.710091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335576960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018718.710099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.710102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018718.710103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018718.710106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.710109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.710112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018718.710116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018718.710124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335577088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018718.710132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018718.710135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018718.710136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018718.710140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2983 00000100:00080000:0.0:1734018718.710143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018718.710146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018718.710149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018718.710157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335577216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018718.710159:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710188:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59c8e00 x1818247335576448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335576448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335576448:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018718.710277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335576576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018718.710301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59c8a80 x1818247335576576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335576576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710337:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335576576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018718.710343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335576704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018718.710358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59c8000 x1818247335576704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335576704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710386:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710390:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335576704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018718.710424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335576832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018718.710438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59c8700 x1818247335576832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335576832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710465:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335576832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018718.710474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018718.710487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59c9f80 x1818247335576960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335576960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335576960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018718.710522:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018718.710534:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710541:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59ca300 x1818247335577088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335577088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018718.710571:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018718.710584:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018718.710590:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018718/real 1734018718] req@ffff8800b59cb480 x1818247335577216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018718.710600:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335577216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018718.710607:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018718.710609:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018718.710612:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018719.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018719.407978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018720.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018720.409990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018721.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018721.411973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018722.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018722.413969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018723.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018723.415961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018723.717806:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.717815:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018723.717820:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.717830:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.717839:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.717847:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018723.717882:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335577344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018723.717916:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.717920:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018723.717922:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.717927:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.717930:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.717933:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018723.717943:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018723.717953:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335577472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018723.717963:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.717966:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018723.717968:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.717972:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.717974:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.717977:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018723.717982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018723.717992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335577600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018723.718001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.718004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018723.718005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.718009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.718012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.718015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018723.718019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018723.718027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335577728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018723.718035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.718038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018723.718039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.718043:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.718045:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.718049:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018723.718053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018723.718060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335577856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018723.718070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.718072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018723.718075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.718079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.718081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.718085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718086:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335577344:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018723.718089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018723.718097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335577984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018723.718104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018723.718107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018723.718109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018723.718112:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2988 00000100:00080000:0.0:1734018723.718114:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018723.718117:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018723.718121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018723.718129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335578112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018723.718232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59cb100 x1818247335577344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718288:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335577344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335577344:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018723.718320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335577472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018723.718341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59caa00 x1818247335577472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335577472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335577472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018723.718380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335577600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018723.718395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59cb480 x1818247335577600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335577600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718421:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335577600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018723.718431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335577728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018723.718445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59ca300 x1818247335577728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335577728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335577728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018723.718510:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018723.718524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59c9f80 x1818247335577856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335577856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018723.718560:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018723.718571:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718577:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59c8700 x1818247335577984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335577984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335577984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018723.718608:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335578112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018723.718620:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018723.718626:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018723/real 1734018723] req@ffff8800b59c8000 x1818247335578112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018723.718636:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335578112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018723.718643:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018723.718645:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018723.718648:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335578112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018724.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018724.417983:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018725.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018725.419967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018726.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018726.421950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018727.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018727.423980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018728.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018728.425977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018728.725825:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.725835:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018728.725840:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.725850:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.725860:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.725867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018728.725903:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335578240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018728.725946:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.725950:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018728.725952:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.725956:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.725960:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.725963:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018728.725973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018728.725984:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335578368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018728.725993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.725996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018728.725998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.726001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.726004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.726007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018728.726011:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018728.726019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335578496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018728.726027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.726031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018728.726033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.726036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.726038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.726042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018728.726046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018728.726054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335578624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018728.726098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.726101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018728.726103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.726107:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.726109:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.726114:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018728.726119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018728.726124:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335578240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018728.726128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335578752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018728.726137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.726140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018728.726142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.726145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.726148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.726151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018728.726155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018728.726163:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335578880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018728.726170:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018728.726174:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018728.726175:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018728.726179:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2993 00000100:00080000:0.0:1734018728.726181:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018728.726184:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018728.726188:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018728.726197:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335579008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018728.726221:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59c8a80 x1818247335578240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726281:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335578240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726298:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335578240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018728.726311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335578368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018728.726330:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726336:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59c8e00 x1818247335578368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726347:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335578368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726355:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335578368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018728.726369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335578496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018728.726384:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726390:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59cbb80 x1818247335578496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335578496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335578496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018728.726449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335578624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018728.726463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59c9180 x1818247335578624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335578624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335578624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018728.726500:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335578752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018728.726512:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59c9500 x1818247335578752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335578752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335578752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018728.726547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335578880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018728.726563:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59cb800 x1818247335578880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335578880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726589:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726592:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335578880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018728.726598:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018728.726610:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018728.726616:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018728/real 1734018728] req@ffff8800b59c9c00 x1818247335579008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018728.726627:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335579008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018728.726635:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018728.726637:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018728.726640:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018729.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018729.427934:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018730.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018730.429985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018731.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018731.431953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018732.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018732.433971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018733.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018733.435974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018733.733807:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.733817:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018733.733821:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.733831:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.733840:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018733.733847:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018733.733881:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335579136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018733.733916:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.733920:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018733.733923:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.733927:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.733930:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018733.733934:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018733.733944:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018733.733954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335579264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018733.733965:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.733968:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018733.733970:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.733973:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.734001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018733.734005:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018733.734010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018733.734021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335579392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018733.734031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.734034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018733.734035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.734039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.734041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018733.734044:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018733.734048:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018733.734056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335579520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018733.734066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.734069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018733.734070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.734074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.734077:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018733.734079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335579136:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018733.734081:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018733.734085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018733.734093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335579648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018733.734101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.734104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018733.734105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.734109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.734111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018733.734114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018733.734118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018733.734126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335579776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018733.734133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018733.734136:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018733.734137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018733.734141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 2998 00000100:00080000:0.0:1734018733.734143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018733.734146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018733.734150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018733.734157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335579904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018733.734173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734205:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59ca680 x1818247335579136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734228:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335579136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335579136:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018733.734258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335579264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018733.734277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734284:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59cad80 x1818247335579264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335579264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734304:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335579264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018733.734316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335579392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018733.734330:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734336:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59c9c00 x1818247335579392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335579392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335579392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018733.734400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335579520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018733.734414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734420:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59cb800 x1818247335579520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335579520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335579520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018733.734450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018733.734463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59c9500 x1818247335579648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335579648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018733.734499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018733.734511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59c9180 x1818247335579776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335579776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018733.734545:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018733.734558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018733.734563:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018733/real 1734018733] req@ffff8800b59cbb80 x1818247335579904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018733.734573:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335579904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018733.734580:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018733.734583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018733.734586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335579904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018734.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018734.438005:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018735.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018735.439990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018736.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018736.442033:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018737.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018737.444007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018738.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018738.445926:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018738.741800:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.741810:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018738.741815:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.741825:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.741834:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018738.741842:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018738.741878:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335580032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018738.741961:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.741965:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018738.741967:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.741972:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.741975:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018738.741979:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018738.741989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018738.742002:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335580160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018738.742012:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.742015:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018738.742017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.742021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.742023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018738.742026:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018738.742032:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018738.742041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335580288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018738.742050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.742053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018738.742055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.742058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.742060:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018738.742064:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018738.742067:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018738.742075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335580416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018738.742085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.742088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018738.742089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.742093:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.742095:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018738.742095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335580032:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018738.742099:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018738.742103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018738.742110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335580544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018738.742119:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.742122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018738.742124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.742128:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.742130:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018738.742133:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018738.742138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018738.742145:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335580672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018738.742153:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018738.742155:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018738.742157:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018738.742160:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3003 00000100:00080000:0.0:1734018738.742163:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018738.742166:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018738.742170:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018738.742178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335580800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018738.742190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59c8e00 x1818247335580032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335580032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742257:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335580032:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018738.742274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335580160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018738.742313:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59c8a80 x1818247335580160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335580160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335580160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018738.742355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335580288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018738.742378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59c8000 x1818247335580288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335580288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335580288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018738.742443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335580416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018738.742456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59c8700 x1818247335580416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742473:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335580416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335580416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018738.742493:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335580544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018738.742505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59c9f80 x1818247335580544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335580544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335580544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018738.742542:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335580672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018738.742554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59ca300 x1818247335580672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335580672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742580:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335580672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018738.742589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335580800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018738.742601:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018738.742607:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018738/real 1734018738] req@ffff8800b59cb480 x1818247335580800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018738.742617:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335580800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018738.742626:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018738.742628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018738.742631:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335580800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018739.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018739.447974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018740.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018740.449958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018741.213889:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018741.451950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018742.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018742.454005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018743.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018743.455983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018743.749870:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.749881:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018743.749885:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.749896:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.749904:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.749912:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018743.749947:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335580928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018743.749987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.749992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018743.749993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.749998:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.750001:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.750005:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018743.750014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018743.750024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335581056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018743.750033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.750037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018743.750038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.750042:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.750045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.750048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018743.750052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018743.750062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335581184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018743.750070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.750074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018743.750075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.750079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.750081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.750085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018743.750089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018743.750097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335581312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018743.750106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.750109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018743.750111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.750115:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.750117:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.750121:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018743.750125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018743.750128:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335580928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018743.750132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335581440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018743.750140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.750143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018743.750144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.750148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.750150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.750153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018743.750157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018743.750165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335581568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018743.750172:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018743.750176:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018743.750178:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018743.750181:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3008 00000100:00080000:0.0:1734018743.750183:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018743.750186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018743.750190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018743.750198:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335581696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018743.750225:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59caa00 x1818247335580928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750278:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335580928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335580928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018743.750308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018743.750326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59cb100 x1818247335581056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750345:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335581056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018743.750365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018743.750381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59cb480 x1818247335581184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750405:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335581184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018743.750453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018743.750466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59ca300 x1818247335581312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335581312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018743.750502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335581440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018743.750515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59c9f80 x1818247335581440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335581440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335581440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018743.750551:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335581568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018743.750564:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750570:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59c8700 x1818247335581568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335581568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750589:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750592:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335581568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018743.750598:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335581696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018743.750610:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018743.750616:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018743/real 1734018743] req@ffff8800b59c8000 x1818247335581696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018743.750626:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335581696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018743.750633:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018743.750635:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018743.750638:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335581696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018744.213882:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018744.457994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018745.213873:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018745.459982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018746.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018746.461997:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018747.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018747.464005:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018748.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018748.466033:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018748.757823:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.757835:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018748.757839:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.757850:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.757859:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.757867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018748.757903:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335581824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018748.757938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.757943:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018748.757944:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.757949:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.757952:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.757955:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018748.757965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018748.757976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335581952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018748.757987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.757990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018748.757992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.757995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.757998:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.758001:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018748.758006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018748.758013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335582080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018748.758022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.758025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018748.758027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.758030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.758033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.758036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018748.758040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018748.758047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335582208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018748.758055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.758058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018748.758059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.758063:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.758066:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.758069:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018748.758073:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018748.758081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335582336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018748.758081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335581824:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018748.758091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.758093:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018748.758095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.758098:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.758100:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.758103:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018748.758107:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018748.758115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335582464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018748.758122:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018748.758125:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018748.758127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018748.758130:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3013 00000100:00080000:0.0:1734018748.758132:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018748.758136:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018748.758140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018748.758148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335582592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018748.758174:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758203:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59c8a80 x1818247335581824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758226:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335581824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758248:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335581824:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018748.758256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018748.758275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59c8e00 x1818247335581952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335581952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335581952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018748.758314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018748.758328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59cbb80 x1818247335582080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758345:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335582080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758354:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018748.758365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018748.758378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758384:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59c9180 x1818247335582208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758394:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335582208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018748.758443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335582336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018748.758457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59c9500 x1818247335582336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335582336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335582336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018748.758494:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335582464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018748.758506:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758512:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59cb800 x1818247335582464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335582464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335582464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018748.758541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335582592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018748.758553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018748.758559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018748/real 1734018748] req@ffff8800b59c9c00 x1818247335582592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018748.758569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335582592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018748.758576:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018748.758579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018748.758582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335582592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018749.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018749.467950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018750.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018750.469932:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018751.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018751.472013:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018752.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018752.473976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018753.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018753.475920:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018753.765840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.765851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018753.765855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.765866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.765874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.765882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018753.765918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335582720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018753.765953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.765957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018753.765959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.765964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.765967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.765970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018753.765980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018753.765991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335582848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018753.766000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.766004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018753.766005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.766009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.766012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.766015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018753.766021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018753.766030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335582976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018753.766039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.766043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018753.766044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.766048:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.766050:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.766054:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018753.766057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018753.766065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335583104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018753.766073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.766076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018753.766078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.766081:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.766084:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.766087:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018753.766091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018753.766099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335583232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018753.766109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.766111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1734018753.766111:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335582720:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018753.766114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.766117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.766120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.766123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018753.766128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018753.766135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335583360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018753.766143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018753.766146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018753.766147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018753.766150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3018 00000100:00080000:0.0:1734018753.766153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018753.766156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018753.766160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018753.766167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335583488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018753.766204:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59cad80 x1818247335582720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335582720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335582720:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018753.766288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018753.766306:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766313:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59ca680 x1818247335582848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335582848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018753.766346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018753.766361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59c9c00 x1818247335582976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335582976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335582976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018753.766400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335583104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018753.766412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59cb800 x1818247335583104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335583104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766435:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766438:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335583104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018753.766476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335583232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018753.766489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59c9500 x1818247335583232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335583232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335583232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018753.766528:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335583360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018753.766540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59c9180 x1818247335583360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335583360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335583360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018753.766576:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335583488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018753.766588:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018753.766594:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018753/real 1734018753] req@ffff8800b59cbb80 x1818247335583488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018753.766604:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335583488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018753.766611:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018753.766614:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018753.766617:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335583488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018754.213790:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018754.477955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018755.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018755.479939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018756.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018756.481927:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018757.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018757.483984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018758.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018758.486011:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018758.773831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.773841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018758.773845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.773856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.773866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018758.773873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018758.773908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335583616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018758.773950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.773954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018758.773956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.773960:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.773963:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018758.773967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018758.773976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018758.773987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335583744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018758.773996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.773999:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018758.774001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.774005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.774007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018758.774011:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018758.774015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018758.774023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335583872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018758.774032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.774035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018758.774036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.774039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.774042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018758.774046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018758.774050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018758.774058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335584000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018758.774067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.774070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018758.774071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.774075:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.774078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018758.774080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335583616:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018758.774081:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018758.774085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018758.774093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335584128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018758.774101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.774104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018758.774105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.774108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.774111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018758.774114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018758.774118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018758.774126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335584256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018758.774133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018758.774137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018758.774139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018758.774142:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3023 00000100:00080000:0.0:1734018758.774144:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018758.774147:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018758.774151:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018758.774160:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335584384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018758.774177:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59c8e00 x1818247335583616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335583616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774246:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335583616:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018758.774260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335583744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018758.774279:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774285:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59c8a80 x1818247335583744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335583744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335583744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018758.774317:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335583872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018758.774332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59c8000 x1818247335583872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335583872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335583872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018758.774397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018758.774410:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774416:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59c8700 x1818247335584000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335584000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774440:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018758.774446:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335584128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018758.774458:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59c9f80 x1818247335584128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335584128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335584128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018758.774495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335584256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018758.774507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59ca300 x1818247335584256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774523:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335584256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335584256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018758.774541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335584384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018758.774604:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018758.774613:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018758/real 1734018758] req@ffff8800b59cb480 x1818247335584384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018758.774623:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335584384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018758.774630:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018758.774633:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018758.774636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335584384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018759.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018759.487943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018760.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018760.489964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018761.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018761.491990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018762.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018762.493915:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018763.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018763.495995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018763.781810:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.781819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018763.781823:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.781834:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.781842:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.781850:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018763.781884:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335584512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018763.781919:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.781923:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018763.781925:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.781929:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.781932:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.781936:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018763.781946:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018763.781956:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335584640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018763.781967:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.781971:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018763.781972:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.781976:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.781978:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.781981:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018763.781986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018763.781995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335584768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018763.782004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.782007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018763.782008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.782012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.782014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.782017:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018763.782021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018763.782029:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335584896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018763.782039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.782042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734018763.782042:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335584512:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018763.782043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.782047:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.782051:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.782054:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018763.782059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018763.782066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335585024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018763.782074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.782077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018763.782079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.782082:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.782084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.782087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018763.782091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018763.782098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335585152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018763.782106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018763.782109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018763.782110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018763.782114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3028 00000100:00080000:0.0:1734018763.782117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018763.782120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018763.782124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018763.782131:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335585280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018763.782135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59cb100 x1818247335584512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782188:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335584512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782202:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782206:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782211:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335584512:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018763.782219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018763.782239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59caa00 x1818247335584640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335584640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018763.782279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018763.782294:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59cb480 x1818247335584768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335584768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018763.782357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018763.782371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59ca300 x1818247335584896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782387:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335584896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335584896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018763.782406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018763.782419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59c9f80 x1818247335585024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335585024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018763.782453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018763.782514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59c8700 x1818247335585152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335585152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018763.782554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018763.782572:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018763.782578:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018763/real 1734018763] req@ffff8800b59c8000 x1818247335585280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018763.782589:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335585280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018763.782597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018763.782599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018763.782602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018764.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018764.497975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018765.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018765.499928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018766.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018766.501980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018767.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018767.503992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018768.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018768.505963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018768.789828:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.789838:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018768.789843:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.789854:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00080000:0.0:1734018768.789863:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.789871:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018768.789906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335585408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018768.789942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.789946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018768.789948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.789952:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00080000:0.0:1734018768.789956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.789959:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018768.789969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018768.789980:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335585536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018768.789990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.789993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018768.789995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.789999:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00080000:0.0:1734018768.790001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.790004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018768.790010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018768.790018:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335585664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018768.790026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.790030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018768.790031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.790035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00080000:0.0:1734018768.790037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.790041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018768.790045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018768.790052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335585792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018768.790060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.790063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018768.790064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.790068:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00080000:0.0:1734018768.790070:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.790073:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018768.790078:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018768.790084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335585920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018768.790094:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.790097:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018768.790099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.790103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00100000:3.0:1734018768.790103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335585408:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018768.790106:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.790109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018768.790113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018768.790121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335586048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018768.790129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018768.790132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018768.790133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018768.790137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3033 00000100:00080000:0.0:1734018768.790139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018768.790142:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018768.790146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018768.790155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335586176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018768.790202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59c8a80 x1818247335585408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790256:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335585408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790269:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335585408:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018768.790286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335585536:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018768.790305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790312:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59c8e00 x1818247335585536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790324:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335585536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335585536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018768.790345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335585664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018768.790360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790366:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59cbb80 x1818247335585664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335585664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335585664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018768.790450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335585792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018768.790468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59c9180 x1818247335585792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335585792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335585792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018768.790539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018768.790552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59c9500 x1818247335585920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335585920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335585920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018768.790591:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018768.790603:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790609:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59cb800 x1818247335586048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790619:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335586048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790626:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790629:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790632:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018768.790637:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018768.790649:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018768.790656:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018768/real 1734018768] req@ffff8800b59c9c00 x1818247335586176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018768.790667:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335586176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018768.790674:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018768.790676:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018768.790679:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018769.213974:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018769.507988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018770.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018770.509963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018771.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018771.511947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018772.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018772.513976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018773.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018773.515976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018773.797811:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.797822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018773.797827:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.797837:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.797847:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.797855:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018773.797892:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335586304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018773.797926:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.797931:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018773.797933:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.797937:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.797941:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.797944:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018773.797954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018773.797965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335586432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018773.797974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.797978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018773.797979:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.797983:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.797985:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.797989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018773.797993:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018773.798003:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335586560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018773.798012:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.798018:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018773.798019:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.798023:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.798026:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.798029:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018773.798033:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018773.798041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335586688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018773.798050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.798053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018773.798055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.798058:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.798061:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.798064:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018773.798068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018773.798073:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335586304:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018773.798076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335586816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018773.798084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.798087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018773.798088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.798092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.798094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.798097:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018773.798103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018773.798111:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335586944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018773.798119:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018773.798122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018773.798124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018773.798127:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3038 00000100:00080000:0.0:1734018773.798129:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018773.798132:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018773.798136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018773.798144:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335587072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018773.798208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335586432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018773.798232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335586560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018773.798262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335586688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018773.798280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018773.798327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018773.798346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018774.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018774.517911:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018775.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018775.519975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018776.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018776.521993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018776.803987:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018776.804032:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018776.804051:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018776.804089:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018776.804100:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018776.804107:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018776.804114:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018776.804121:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018776.804128:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018776.804230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59cb800 x1818247335586688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335586688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335586688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018776.804334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59c9500 x1818247335586816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335586816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018776.804371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59c9180 x1818247335586944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335586944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804397:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335586944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018776.804404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59cbb80 x1818247335587072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335587072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018776.804435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59ca680 x1818247335586304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804447:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335586304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335586304:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018776.804476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59cad80 x1818247335586432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335586432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335586432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018776.804509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018773/real 1734018776] req@ffff8800b59c9c00 x1818247335586560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018776.804527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335586560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018776.804538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018776.804542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018776.804547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335586560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018777.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018777.523955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018778.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018778.525960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018778.805838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.805850:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018778.805855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.805868:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.805879:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.805889:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018778.805936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335587200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018778.805981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.805986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018778.805988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.805993:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.805997:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.806000:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018778.806012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018778.806024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335587328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018778.806034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.806037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018778.806039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.806043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.806045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.806049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018778.806054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018778.806062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335587456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018778.806070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.806074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018778.806075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.806079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.806081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.806086:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335587200:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018778.806090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018778.806099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335587584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018778.806108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.806112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018778.806113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.806117:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.806119:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.806123:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018778.806127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018778.806135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335587712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018778.806143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.806146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018778.806147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.806151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.806154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.806157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018778.806161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018778.806172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335587840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018778.806173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018778.806181:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018778.806185:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018778.806186:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018778.806190:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3043 00000100:00080000:0.0:1734018778.806193:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018778.806196:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018778.806200:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018778.806201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59c8e00 x1818247335587200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018778.806239:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335587968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018778.806265:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335587200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335587200:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018778.806295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335587328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018778.806315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806323:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59c8a80 x1818247335587328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018778.806334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335587328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335587328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018778.806355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335587456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018778.806369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59c8000 x1818247335587456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018778.806385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335587456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335587456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018778.806430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335587584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018778.806443:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59c8700 x1818247335587584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018778.806460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335587584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335587584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018778.806478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018778.806490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59c9f80 x1818247335587712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018778.806506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335587712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018778.806525:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018778.806540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59ca300 x1818247335587840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018778.806557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335587840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018778.806574:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018778.806586:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018778.806592:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018778/real 1734018778] req@ffff8800b59cb480 x1818247335587968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018778.806602:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335587968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018778.806609:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018778.806611:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018778.806614:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335587968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018779.213927:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018779.528007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018780.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018780.529959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018781.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018781.531973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018782.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018782.533961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018783.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018783.535956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018783.813829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.813838:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018783.813842:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.813853:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.813862:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.813870:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018783.813906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335588096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018783.813942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.813947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018783.813949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.813953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.813956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.813960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018783.813970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018783.813980:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335588224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018783.813992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.813995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018783.813996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.814000:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.814002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.814006:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018783.814010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018783.814020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335588352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018783.814028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.814031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018783.814033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.814036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.814039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.814043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018783.814047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018783.814055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335588480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018783.814063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.814066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018783.814068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.814071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.814074:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.814077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018783.814081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018783.814089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335588608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018783.814098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.814101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018783.814102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.814106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.814108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.814111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018783.814115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018783.814118:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335588096:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018783.814123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335588736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018783.814170:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018783.814173:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018783.814175:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018783.814179:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3048 00000100:00080000:0.0:1734018783.814181:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018783.814184:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018783.814189:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018783.814198:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335588864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018783.814260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59caa00 x1818247335588096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335588096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814337:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335588096:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018783.814345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335588224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018783.814364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59cb100 x1818247335588224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335588224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814397:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335588224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018783.814403:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335588352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018783.814418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59cb480 x1818247335588352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335588352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335588352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018783.814463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335588480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018783.814476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59ca300 x1818247335588480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335588480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335588480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018783.814540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335588608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018783.814553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59c9f80 x1818247335588608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335588608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335588608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018783.814591:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335588736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018783.814604:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814610:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59c8700 x1818247335588736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335588736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814628:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814630:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814633:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335588736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018783.814639:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335588864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018783.814651:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018783.814657:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018783/real 1734018783] req@ffff8800b59c8000 x1818247335588864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018783.814667:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335588864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018783.814675:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018783.814677:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018783.814680:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335588864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018784.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018784.537994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018785.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018785.539978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018786.213877:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018786.541961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018787.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018787.544002:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018788.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018788.546000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018788.821820:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.821830:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018788.821835:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.821846:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.821855:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.821863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018788.821899:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335588992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018788.821934:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.821938:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018788.821940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.821944:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.821947:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.821951:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018788.821961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018788.821971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335589120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018788.821981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.821984:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018788.821985:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.821989:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.821992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.821995:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018788.822001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018788.822009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335589248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018788.822018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.822021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018788.822023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.822026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.822028:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.822032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018788.822036:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018788.822072:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335589376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018788.822083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.822086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018788.822087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.822091:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.822093:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.822097:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018788.822102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018788.822104:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335588992:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018788.822109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335589504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018788.822117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.822120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018788.822122:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.822126:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.822129:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.822132:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018788.822136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018788.822143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335589632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018788.822151:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018788.822154:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018788.822155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018788.822158:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3053 00000100:00080000:0.0:1734018788.822161:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018788.822164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018788.822168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018788.822176:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335589760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018788.822200:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822231:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59c8a80 x1818247335588992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335588992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335588992:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018788.822283:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335589120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018788.822301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59c8e00 x1818247335589120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335589120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335589120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018788.822341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335589248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018788.822356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59c9c00 x1818247335589248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335589248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822389:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822395:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335589248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018788.822428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335589376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018788.822441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59cad80 x1818247335589376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335589376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335589376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018788.822477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335589504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018788.822490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59ca680 x1818247335589504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822505:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335589504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335589504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018788.822526:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335589632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018788.822538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59cbb80 x1818247335589632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335589632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822561:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335589632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018788.822572:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335589760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018788.822585:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018788.822591:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018788/real 1734018788] req@ffff8800b59c9180 x1818247335589760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018788.822601:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335589760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018788.822608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018788.822610:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018788.822613:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335589760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018789.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018789.547932:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018790.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018790.549983:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018791.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018791.552045:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018792.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018792.553938:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018793.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018793.556022:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018793.829821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.829830:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018793.829835:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.829846:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.829856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018793.829864:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018793.829899:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335589888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018793.829942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.829946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018793.829948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.829979:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.829983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018793.829987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018793.829997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018793.830009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335590016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018793.830019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.830023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018793.830025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.830029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.830031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018793.830034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018793.830039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018793.830048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335590144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018793.830058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.830061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018793.830063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.830067:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.830069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018793.830074:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018793.830078:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018793.830080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335589888:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018793.830087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335590272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018793.830095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.830098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018793.830099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.830103:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.830105:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018793.830109:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018793.830113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018793.830120:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335590400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018793.830128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.830131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018793.830132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.830136:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.830138:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018793.830141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018793.830145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018793.830153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335590528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018793.830160:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018793.830164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018793.830165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018793.830169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3058 00000100:00080000:0.0:1734018793.830172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018793.830175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018793.830176:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018793.830179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018793.830187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335590656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018793.830207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59c9500 x1818247335589888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335589888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830252:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335589888:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018793.830260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018793.830280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018793.830287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59cb800 x1818247335590016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335590016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018793.830342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018793.830357:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018793.830363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59c9180 x1818247335590144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335590144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830386:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018793.830395:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018793.830407:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018793.830413:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59cbb80 x1818247335590272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830423:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335590272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018793.830442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335590400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018793.830456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018793.830462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59ca680 x1818247335590400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335590400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335590400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018793.830492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335590528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018793.830505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018793.830511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59cad80 x1818247335590528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335590528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335590528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018793.830539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335590656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018793.830552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018793.830558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018793/real 1734018793] req@ffff8800b59c9c00 x1818247335590656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018793.830568:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335590656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018793.830576:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018793.830578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018793.830582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335590656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018794.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018794.557987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018795.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018795.559903:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018796.213895:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018796.562050:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018797.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018797.564014:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018798.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018798.565962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018798.837844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.837855:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018798.837859:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.837909:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.837920:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.837928:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018798.837965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335590784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018798.838001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.838005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018798.838007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.838012:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.838015:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.838018:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018798.838028:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018798.838039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335590912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018798.838050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.838053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018798.838055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.838058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.838061:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.838064:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018798.838068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018798.838076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335591040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018798.838086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.838088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018798.838090:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.838093:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.838096:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.838099:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018798.838103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018798.838110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335591168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018798.838120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.838123:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018798.838125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.838129:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.838132:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.838136:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018798.838140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018798.838143:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335590784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018798.838148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335591296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018798.838156:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.838159:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018798.838161:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.838164:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.838167:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.838170:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018798.838174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018798.838181:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335591424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018798.838188:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018798.838191:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018798.838193:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018798.838196:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3063 00000100:00080000:0.0:1734018798.838198:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018798.838201:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018798.838205:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018798.838214:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335591552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018798.838239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838270:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59c8e00 x1818247335590784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335590784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838304:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335590784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018798.838321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590912:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018798.838340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59c8a80 x1818247335590912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335590912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335590912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018798.838379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018798.838394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59c8000 x1818247335591040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838412:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335591040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838421:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838423:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838426:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018798.838460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018798.838473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59c8700 x1818247335591168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335591168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018798.838509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335591296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018798.838521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59c9f80 x1818247335591296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335591296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838548:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335591296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018798.838558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335591424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018798.838570:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838576:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59ca300 x1818247335591424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838586:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335591424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838593:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838596:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335591424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018798.838604:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335591552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018798.838617:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018798.838623:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018798/real 1734018798] req@ffff8800b59cb480 x1818247335591552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018798.838632:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335591552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018798.838640:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018798.838642:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018798.838645:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335591552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018799.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018799.568025:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018800.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018800.569939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018801.213898:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018801.571985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018802.213874:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018802.574003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018803.213882:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018803.575992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018803.845885:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.845896:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018803.845900:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018803.845910:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.845920:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.845928:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018803.845965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335591680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018803.845999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.846003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018803.846005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018803.846009:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.846013:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.846016:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018803.846027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018803.846037:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335591808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018803.846047:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.846050:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018803.846051:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018803.846055:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.846058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.846061:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018803.846067:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018803.846077:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335591936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018803.846086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.846089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018803.846091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018803.846095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.846097:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.846101:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018803.846105:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018803.846113:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335592064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018803.846123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.846126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018803.846127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018803.846130:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335591680:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018803.846131:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.846134:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.846137:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018803.846142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018803.846149:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335592192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018803.846157:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.846160:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018803.846162:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018803.846166:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.846168:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.846171:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018803.846176:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018803.846183:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335592320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018803.846190:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018803.846193:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018803.846195:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018803.846198:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3068 00000100:00080000:0.0:1734018803.846201:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018803.846204:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018803.846207:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018803.846215:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335592448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018803.846226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59cb100 x1818247335591680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335591680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335591680:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018803.846309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018803.846327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59caa00 x1818247335591808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335591808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846354:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018803.846366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018803.846381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59cb480 x1818247335591936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335591936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846410:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335591936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018803.846442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018803.846455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59ca300 x1818247335592064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335592064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018803.846492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335592192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018803.846504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59c9f80 x1818247335592192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335592192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335592192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018803.846541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335592320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018803.846554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59c8700 x1818247335592320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335592320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335592320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018803.846588:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335592448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018803.846602:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018803.846608:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018803/real 1734018803] req@ffff8800b59c8000 x1818247335592448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018803.846618:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335592448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018803.846626:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018803.846628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018803.846631:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335592448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018804.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018804.577988:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018805.213923:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018805.579978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018806.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018806.581993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018807.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018807.583962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018808.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018808.585994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018808.853829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018808.853841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018808.853846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.853861:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.853872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.853883:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018808.853931:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335592576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018808.853977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018808.853982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018808.853984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.853989:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.853993:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.853997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018808.854009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018808.854020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335592704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018808.854030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018808.854033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018808.854035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.854039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.854042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.854046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018808.854051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018808.854059:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335592832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018808.854067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018808.854070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018808.854072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.854076:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.854078:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.854083:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018808.854087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018808.854092:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335592576:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018808.854095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335592960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018808.854104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018808.854107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018808.854109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.854113:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.854115:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.854119:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018808.854123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018808.854130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335593088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018808.854138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018808.854141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018808.854142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.854146:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.854149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.854152:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018808.854157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018808.854165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335593216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018808.854173:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734018808.854176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018808.854177:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018808.854179:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018808.854183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3073 00000100:00080000:0.0:1734018808.854186:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018808.854189:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018808.854193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018808.854203:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335593344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018808.854204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59c8a80 x1818247335592576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854225:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335592576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854236:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854240:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854245:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335592576:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018808.854253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592704:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018808.854270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018808.854277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59c8e00 x1818247335592704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335592704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854297:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018808.854309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018808.854323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018808.854329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59c9c00 x1818247335592832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335592832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018808.854387:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018808.854400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018808.854406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59cad80 x1818247335592960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335592960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854424:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335592960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018808.854435:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018808.854447:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018808.854453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59ca680 x1818247335593088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335593088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018808.854483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018808.854496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018808.854501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59cbb80 x1818247335593216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335593216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018808.854528:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018808.854540:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018808.854546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018808/real 1734018808] req@ffff8800b59c9180 x1818247335593344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018808.854556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335593344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018808.854563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018808.854565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018808.854568:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018809.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018809.587992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018810.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018810.589948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018811.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018811.591981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018812.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018812.594000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018813.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018813.595969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018813.861810:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.861820:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018813.861824:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.861836:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.861846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.861854:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018813.861891:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335593472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018813.861927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.861931:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018813.861933:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.861938:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.861941:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.861944:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018813.861954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018813.861965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335593600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018813.861976:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.861979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018813.861981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.861984:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.861987:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.861990:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018813.861994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018813.862004:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335593728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018813.862012:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.862016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018813.862017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.862021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.862023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.862026:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018813.862030:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018813.862038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335593856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018813.862046:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.862049:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018813.862051:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.862054:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.862057:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.862061:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018813.862065:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018813.862073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335593984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018813.862079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335593472:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018813.862083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.862086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018813.862087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.862091:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.862093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.862096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018813.862100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018813.862108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335594112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018813.862116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018813.862119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018813.862120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018813.862124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3078 00000100:00080000:0.0:1734018813.862126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018813.862129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018813.862133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018813.862140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335594240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018813.862175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59cb800 x1818247335593472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335593472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335593472:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018813.862258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335593600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018813.862277:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862284:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59c9500 x1818247335593600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335593600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335593600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018813.862315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335593728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018813.862328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59c9180 x1818247335593728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335593728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862354:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335593728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018813.862365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335593856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018813.862377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59cbb80 x1818247335593856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335593856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335593856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018813.862441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018813.862454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59ca680 x1818247335593984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335593984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335593984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018813.862492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335594112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018813.862505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59cad80 x1818247335594112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335594112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335594112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018813.862540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335594240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018813.862552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018813.862558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018813/real 1734018813] req@ffff8800b59c9c00 x1818247335594240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018813.862567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335594240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018813.862575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018813.862577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018813.862580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335594240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018814.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018814.597952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018815.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018815.599964:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018816.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018816.601844:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018817.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018817.603971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018818.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018818.605963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018818.869848:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.869859:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018818.869864:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.869877:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.869888:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.869899:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018818.869948:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335594368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018818.869987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.869992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018818.869994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.869999:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.870002:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.870006:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018818.870018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018818.870029:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335594496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018818.870039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.870042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018818.870044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.870049:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.870051:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.870055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018818.870061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018818.870070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335594624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018818.870079:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.870082:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018818.870084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.870087:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.870090:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.870094:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335594368:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018818.870098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018818.870107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335594752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018818.870115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.870119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018818.870120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.870124:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.870126:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.870130:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018818.870134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018818.870141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335594880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018818.870150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.870153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018818.870155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.870159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.870162:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.870166:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018818.870170:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018818.870178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335595008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018818.870181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018818.870185:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018818.870188:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018818.870190:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018818.870194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3083 00000100:00080000:0.0:1734018818.870196:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018818.870199:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018818.870203:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018818.870208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59c8e00 x1818247335594368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018818.870214:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335595136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018818.870230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335594368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335594368:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018818.870257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335594496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018818.870275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59c8a80 x1818247335594496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018818.870292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335594496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335594496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018818.870313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335594624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018818.870327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59c8000 x1818247335594624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018818.870344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335594624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335594624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018818.870389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335594752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018818.870402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59c8700 x1818247335594752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018818.870418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335594752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335594752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018818.870436:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335594880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018818.870448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870454:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59c9f80 x1818247335594880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018818.870463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335594880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335594880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018818.870483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018818.870494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59ca300 x1818247335595008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018818.870510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335595008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870522:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018818.870527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018818.870588:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018818.870595:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018818/real 1734018818] req@ffff8800b59cb480 x1818247335595136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018818.870605:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335595136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018818.870613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018818.870615:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018818.870618:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018819.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018819.607964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018820.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018820.609948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018821.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018821.611989:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018822.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018822.613965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018823.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018823.615993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018823.877848:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.877858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018823.877862:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.877873:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.877882:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018823.877890:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018823.877926:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335595264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018823.877968:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.877972:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018823.877974:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.877979:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.877982:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018823.877985:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018823.877995:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018823.878005:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335595392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018823.878014:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.878017:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018823.878019:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.878023:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.878025:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018823.878029:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018823.878033:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018823.878043:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335595520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018823.878052:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.878054:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018823.878056:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.878060:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.878062:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018823.878066:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018823.878070:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018823.878078:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335595648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018823.878088:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.878091:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018823.878092:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.878096:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.878099:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018823.878101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335595264:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018823.878102:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018823.878107:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018823.878114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335595776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018823.878122:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.878125:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018823.878127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.878131:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.878133:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018823.878136:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018823.878141:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018823.878148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335595904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018823.878155:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018823.878159:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018823.878160:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018823.878164:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3088 00000100:00080000:0.0:1734018823.878167:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018823.878170:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018823.878174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018823.878182:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335596032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018823.878195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59caa00 x1818247335595264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335595264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335595264:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018823.878275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335595392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018823.878294:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59cb100 x1818247335595392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335595392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335595392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018823.878332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335595520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018823.878347:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878353:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59cb480 x1818247335595520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335595520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335595520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018823.878410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335595648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018823.878423:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59ca300 x1818247335595648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335595648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335595648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018823.878502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018823.878519:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59c9f80 x1818247335595776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878538:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335595776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018823.878559:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018823.878572:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878578:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59c8700 x1818247335595904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335595904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878596:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335595904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018823.878608:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018823.878621:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018823.878627:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018823/real 1734018823] req@ffff8800b59c8000 x1818247335596032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018823.878637:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335596032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018823.878644:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018823.878646:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018823.878650:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018824.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018824.617995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018825.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018825.619986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018826.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018826.621985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018827.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018827.623997:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018828.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018828.625964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018828.885822:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.885832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018828.885836:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.885847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.885856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018828.885863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018828.885899:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335596160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018828.885933:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.885937:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018828.885939:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.885944:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.885947:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018828.885950:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018828.885960:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018828.885971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335596288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018828.885982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.885985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018828.885986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.885990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.885992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018828.885996:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018828.886000:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018828.886008:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335596416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018828.886016:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.886019:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018828.886021:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.886025:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.886027:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018828.886030:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018828.886034:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018828.886042:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335596544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018828.886050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.886053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018828.886055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.886058:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.886062:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018828.886065:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018828.886069:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018828.886076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335596672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018828.886085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.886088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018828.886089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.886093:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.886095:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018828.886096:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335596160:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018828.886099:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018828.886103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018828.886110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335596800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018828.886118:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018828.886120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018828.886122:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018828.886125:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3093 00000100:00080000:0.0:1734018828.886127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018828.886130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018828.886134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018828.886143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335596928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018828.886196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886227:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59c8a80 x1818247335596160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335596160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335596160:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018828.886280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335596288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018828.886299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59c8e00 x1818247335596288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335596288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335596288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018828.886338:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335596416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018828.886353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59c9c00 x1818247335596416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335596416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335596416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018828.886439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335596544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018828.886456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59cad80 x1818247335596544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886473:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335596544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335596544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018828.886524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018828.886538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59ca680 x1818247335596672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335596672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018828.886575:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018828.886588:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886593:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59cbb80 x1818247335596800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886603:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335596800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886611:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886613:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886616:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018828.886622:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018828.886634:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018828.886640:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018828/real 1734018828] req@ffff8800b59c9180 x1818247335596928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018828.886651:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335596928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018828.886658:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018828.886661:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018828.886664:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335596928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018829.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018829.627969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018830.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018830.629973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018831.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018831.631994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018832.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018832.633979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018833.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018833.635989:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018833.893835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.893847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018833.893851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.893862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.893872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.893879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018833.893916:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335597056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018833.893951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.893955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018833.893957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.893962:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.893965:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.893968:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018833.893978:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018833.893989:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335597184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018833.893998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.894001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018833.894003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.894007:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.894009:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.894013:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018833.894019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018833.894029:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335597312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018833.894038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.894041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018833.894043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.894046:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.894048:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.894052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018833.894056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018833.894063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335597440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018833.894073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.894076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018833.894077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.894081:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.894083:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.894087:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018833.894091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018833.894091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335597056:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018833.894099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335597568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018833.894108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.894111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018833.894112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.894116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.894119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.894122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018833.894127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018833.894134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335597696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018833.894142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018833.894145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018833.894146:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018833.894149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3098 00000100:00080000:0.0:1734018833.894152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018833.894155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018833.894158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018833.894166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335597824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018833.894187:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59c9500 x1818247335597056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894240:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335597056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335597056:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018833.894269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335597184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018833.894336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59cb800 x1818247335597184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335597184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894371:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335597184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018833.894377:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335597312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018833.894393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59c9180 x1818247335597312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335597312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335597312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018833.894457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335597440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018833.894471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59cbb80 x1818247335597440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335597440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335597440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018833.894508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335597568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018833.894520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59ca680 x1818247335597568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335597568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335597568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018833.894556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335597696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018833.894569:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894575:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59cad80 x1818247335597696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894584:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335597696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894592:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894594:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335597696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018833.894603:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335597824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018833.894616:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018833.894622:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018833/real 1734018833] req@ffff8800b59c9c00 x1818247335597824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018833.894632:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335597824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018833.894640:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018833.894642:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018833.894645:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335597824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018834.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018834.637995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018835.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018835.639979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018836.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018836.641942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018837.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018837.643974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018838.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018838.645993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018838.901841:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.901852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018838.901857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.901871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.901883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.901893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018838.901941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335597952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018838.901988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.901993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018838.901995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.902000:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.902004:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.902007:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018838.902019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018838.902031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335598080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018838.902041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.902045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018838.902046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.902050:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.902053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.902056:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018838.902061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018838.902069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335598208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018838.902077:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.902080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018838.902082:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.902085:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.902088:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.902092:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018838.902096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018838.902104:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335597952:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018838.902105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335598336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018838.902114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.902118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018838.902119:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.902123:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.902126:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.902129:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018838.902134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018838.902141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335598464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018838.902149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.902152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018838.902154:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.902157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.902160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.902163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018838.902167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018838.902175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335598592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018838.902184:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018838.902224:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018838.902226:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018838.902231:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3103 00000100:00080000:0.0:1734018838.902234:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018838.902238:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018838.902243:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018838.902248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734018838.902256:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335598720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018838.902273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018838.902300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018838.902315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335598464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018838.902333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335598592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018838.902348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335598720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018839.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018839.647972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018840.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018840.649944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018841.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018841.651927:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018841.907938:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018841.907980:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018841.908000:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018841.908038:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018841.908048:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018841.908055:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018841.908063:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018841.908070:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018841.908076:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018841.908207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59c8700 x1818247335598336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908256:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335598336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908277:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018841.908307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59c9f80 x1818247335598464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335598464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335598464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018841.908385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59ca300 x1818247335598592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908404:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335598592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908422:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335598592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018841.908441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59cb480 x1818247335598720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335598720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335598720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018841.908493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59c8e00 x1818247335597952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335597952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335597952:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018841.908558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59c8a80 x1818247335598080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335598080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018841.908609:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018838/real 1734018841] req@ffff8800b59c8000 x1818247335598208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018841.908641:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335598208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018841.908655:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018841.908660:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018841.908665:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018842.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018842.653976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018843.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018843.655941:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018843.909849:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018843.909862:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018843.909867:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.909882:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.909894:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018843.909905:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018843.909953:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335598848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018843.909991:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018843.909996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018843.909998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.910004:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.910007:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018843.910010:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018843.910022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018843.910034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335598976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018843.910045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018843.910048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018843.910050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.910054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.910056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018843.910060:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018843.910064:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018843.910075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335599104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018843.910084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018843.910087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018843.910088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.910092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.910094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018843.910098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018843.910130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018843.910142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335599232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018843.910152:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734018843.910154:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335598848:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018843.910156:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018843.910158:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.910162:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.910166:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018843.910170:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018843.910175:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018843.910183:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335599360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018843.910191:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018843.910195:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018843.910196:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.910200:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.910202:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018843.910206:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018843.910210:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018843.910218:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335599488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018843.910226:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018843.910229:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018843.910230:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018843.910234:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3108 00000100:00080000:0.0:1734018843.910236:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018843.910239:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018843.910240:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018843.910244:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018843.910252:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335599616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018843.910268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59cb100 x1818247335598848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335598848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335598848:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018843.910319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018843.910336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018843.910343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59caa00 x1818247335598976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335598976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335598976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018843.910375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335599104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018843.910390:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018843.910396:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59c8000 x1818247335599104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335599104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910418:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335599104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018843.910451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335599232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018843.910465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018843.910471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59c8a80 x1818247335599232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335599232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335599232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018843.910500:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335599360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018843.910511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018843.910517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59c8e00 x1818247335599360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335599360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335599360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018843.910545:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335599488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018843.910561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018843.910567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59cb480 x1818247335599488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335599488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910585:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910587:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910590:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335599488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018843.910595:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335599616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018843.910607:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018843.910612:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018843/real 1734018843] req@ffff8800b59ca300 x1818247335599616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018843.910622:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335599616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018843.910630:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018843.910632:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018843.910635:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335599616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018844.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018844.657958:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018845.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018845.659984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018846.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018846.661954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018847.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018847.663996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018848.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018848.666002:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018848.917854:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.917865:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018848.917869:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.917880:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.917890:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018848.917898:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018848.917934:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335599744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018848.917970:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.917974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018848.917976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.917980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.917983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018848.917987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018848.917997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018848.918007:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335599872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018848.918017:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.918059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018848.918061:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.918065:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.918069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018848.918072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018848.918079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018848.918088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335600000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018848.918099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.918102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018848.918104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.918107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.918110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018848.918113:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018848.918118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018848.918118:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335599744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018848.918126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335600128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018848.918134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.918137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018848.918139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.918143:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.918145:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018848.918149:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018848.918152:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018848.918160:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335600256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018848.918168:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.918171:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018848.918173:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.918177:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.918179:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018848.918182:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018848.918186:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018848.918194:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335600384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018848.918201:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018848.918204:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018848.918206:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018848.918209:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3113 00000100:00080000:0.0:1734018848.918212:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734018848.918214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018848.918215:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018848.918220:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018848.918229:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335600512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018848.918246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59c9f80 x1818247335599744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335599744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335599744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018848.918300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335599872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018848.918319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018848.918326:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59c8700 x1818247335599872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335599872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918346:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918352:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335599872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018848.918384:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018848.918399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018848.918405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59c9c00 x1818247335600000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335600000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918431:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918434:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018848.918444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018848.918456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018848.918462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59cad80 x1818247335600128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918472:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335600128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918480:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018848.918492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335600256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018848.918506:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018848.918512:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59ca680 x1818247335600256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335600256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335600256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018848.918541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335600384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018848.918555:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018848.918560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59cbb80 x1818247335600384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335600384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918580:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335600384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018848.918589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335600512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018848.918601:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018848.918607:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018848/real 1734018848] req@ffff8800b59c9180 x1818247335600512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018848.918617:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335600512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018848.918624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018848.918626:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018848.918629:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335600512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018849.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018849.667955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018850.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018850.670032:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018851.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018851.671962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018852.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018852.674008:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018853.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018853.676005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018853.925843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.925854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018853.925858:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.925870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.925880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.925888:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018853.925921:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335600640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018853.926002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.926007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018853.926008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.926013:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.926017:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.926021:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018853.926030:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018853.926042:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335600768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018853.926052:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.926055:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018853.926057:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.926060:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.926063:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.926066:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018853.926070:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018853.926080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335600896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018853.926088:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.926091:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018853.926093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.926096:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.926099:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.926103:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018853.926107:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018853.926114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335601024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018853.926123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.926127:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018853.926128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.926132:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.926134:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.926137:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926138:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335600640:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018853.926142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018853.926149:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335601152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018853.926157:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.926160:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018853.926162:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.926165:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.926168:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.926171:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018853.926175:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018853.926183:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335601280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018853.926191:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018853.926194:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018853.926196:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018853.926199:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3118 00000100:00080000:0.0:1734018853.926204:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018853.926207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018853.926211:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018853.926219:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335601408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018853.926236:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59cb800 x1818247335600640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926287:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335600640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926305:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335600640:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018853.926318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018853.926336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59c9500 x1818247335600768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335600768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926369:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018853.926375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018853.926389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926396:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59c9180 x1818247335600896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335600896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926420:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335600896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018853.926452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018853.926465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59cbb80 x1818247335601024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335601024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018853.926502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335601152:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018853.926515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59ca680 x1818247335601152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335601152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335601152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018853.926550:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335601280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018853.926562:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926568:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59cad80 x1818247335601280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926578:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335601280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926586:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926588:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926591:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335601280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018853.926596:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335601408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018853.926609:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018853.926615:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018853/real 1734018853] req@ffff8800b59c9c00 x1818247335601408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018853.926625:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335601408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018853.926631:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018853.926633:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018853.926637:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335601408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018854.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018854.678003:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018855.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018855.679843:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018856.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018856.681972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018857.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018857.683999:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018858.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018858.685931:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018858.933831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018858.933842:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018858.933846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.933885:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.933895:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.933903:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018858.933941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335601536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018858.933975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018858.933979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018858.933981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.933986:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.933989:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.933992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018858.934002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018858.934013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335601664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018858.934024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018858.934027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018858.934029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.934032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.934035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.934038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018858.934043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018858.934050:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335601792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018858.934058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018858.934062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018858.934063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.934067:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.934069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.934072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018858.934076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018858.934084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335601920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018858.934092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018858.934095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018858.934096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.934099:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.934102:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.934106:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018858.934110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018858.934118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335602048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018858.934127:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734018858.934128:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335601536:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018858.934130:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018858.934132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.934135:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.934137:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.934141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018858.934144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018858.934152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335602176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018858.934160:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018858.934163:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018858.934164:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018858.934167:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3123 00000100:00080000:0.0:1734018858.934170:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018858.934173:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018858.934177:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018858.934186:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335602304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018858.934230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59c8700 x1818247335601536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335601536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335601536:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018858.934313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018858.934331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59c9f80 x1818247335601664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335601664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018858.934371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018858.934386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59ca300 x1818247335601792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335601792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934418:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018858.934423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018858.934436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59cb480 x1818247335601920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335601920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335601920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018858.934501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018858.934515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59c8e00 x1818247335602048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335602048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934543:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934546:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018858.934553:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018858.934565:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934572:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59c8a80 x1818247335602176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934582:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335602176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934590:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018858.934600:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018858.934612:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018858.934618:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018858/real 1734018858] req@ffff8800b59c8000 x1818247335602304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018858.934628:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335602304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018858.934635:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018858.934637:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018858.934640:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018859.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018859.688020:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018860.213929:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018860.689982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018861.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018861.691984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018862.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018862.693996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018863.213947:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018863.696015:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018863.941861:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.941872:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018863.941876:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018863.941886:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.941895:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.941902:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018863.941939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335602432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018863.941972:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.941976:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018863.941978:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018863.941982:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.941986:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.941989:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018863.941999:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018863.942009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335602560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018863.942019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.942022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018863.942024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018863.942027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.942031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.942034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018863.942039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018863.942049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335602688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018863.942058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.942061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018863.942062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018863.942066:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.942069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.942072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018863.942076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018863.942084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335602816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018863.942093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.942096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018863.942098:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018863.942098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335602432:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018863.942101:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.942104:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.942107:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018863.942111:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018863.942119:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335602944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018863.942134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.942137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018863.942138:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018863.942142:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.942144:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.942148:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018863.942151:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018863.942158:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335603072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018863.942166:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018863.942169:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018863.942170:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018863.942173:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3128 00000100:00080000:0.0:1734018863.942176:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018863.942179:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018863.942183:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018863.942191:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335603200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018863.942193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59caa00 x1818247335602432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335602432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335602432:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018863.942277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335602560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018863.942296:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59cb100 x1818247335602560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335602560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335602560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018863.942335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335602688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018863.942350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59c8000 x1818247335602688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335602688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942386:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942389:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335602688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018863.942420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335602816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018863.942433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59c8a80 x1818247335602816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335602816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335602816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018863.942469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018863.942481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59c8e00 x1818247335602944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335602944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335602944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018863.942518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018863.942530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59cb480 x1818247335603072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335603072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942555:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018863.942564:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018863.942577:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018863.942582:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018863/real 1734018863] req@ffff8800b59ca300 x1818247335603200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018863.942592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335603200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018863.942600:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018863.942602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018863.942605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018864.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018864.697965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018865.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018865.699976:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018866.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018866.701966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018867.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018867.703955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018868.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018868.705970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018868.949821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.949832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018868.949837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.949847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.949856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018868.949864:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018868.949899:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335603328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018868.949933:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.949937:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018868.949939:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.949944:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.949947:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018868.949950:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018868.949960:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018868.949970:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335603456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018868.949980:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.949983:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018868.949985:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.949988:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.949991:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018868.949994:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018868.949999:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018868.950006:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335603584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018868.950015:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.950018:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018868.950020:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.950024:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.950026:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018868.950029:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018868.950033:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018868.950041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335603712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018868.950050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.950053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018868.950055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.950059:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.950061:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018868.950061:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335603328:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018868.950064:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018868.950068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018868.950076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335603840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018868.950084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.950086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018868.950088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.950091:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.950094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018868.950097:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018868.950101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018868.950108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335603968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018868.950117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018868.950120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018868.950121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018868.950124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3133 00000100:00080000:0.0:1734018868.950127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018868.950130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018868.950134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018868.950142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335604096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018868.950158:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950188:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59c9f80 x1818247335603328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335603328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335603328:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018868.950242:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335603456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018868.950260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59c8700 x1818247335603456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335603456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950291:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335603456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018868.950300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335603584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018868.950315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59c9c00 x1818247335603584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335603584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950344:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335603584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018868.950380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335603712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018868.950394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59cad80 x1818247335603712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335603712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335603712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018868.950430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018868.950442:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950448:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59ca680 x1818247335603840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950458:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335603840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018868.950478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018868.950490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59cbb80 x1818247335603968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335603968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335603968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018868.950524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018868.950537:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018868.950543:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018868/real 1734018868] req@ffff8800b59c9180 x1818247335604096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018868.950552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335604096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018868.950559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018868.950561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018868.950565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018869.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018869.707981:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018870.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018870.709964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018871.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018871.711959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018872.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018872.713975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018873.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018873.715876:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018873.957835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.957844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018873.957849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.957859:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.957869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018873.957877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018873.957913:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335604224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018873.957948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.957952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018873.957954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.957959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.957962:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018873.957965:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018873.957976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018873.957987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335604352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018873.957997:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.958000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018873.958002:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.958005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.958008:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018873.958012:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018873.958016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018873.958025:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335604480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018873.958033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.958037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018873.958039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.958042:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.958045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018873.958048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018873.958052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018873.958059:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335604608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018873.958067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.958070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018873.958072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.958075:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.958078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018873.958082:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018873.958086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018873.958093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335604736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018873.958102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.958105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018873.958107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.958111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.958113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018873.958113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335604224:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018873.958116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018873.958120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018873.958128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335604864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018873.958136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018873.958139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018873.958140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018873.958144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3138 00000100:00080000:0.0:1734018873.958146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018873.958149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018873.958153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018873.958161:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335604992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018873.958211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958243:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59c9500 x1818247335604224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958265:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335604224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958279:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335604224:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018873.958296:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335604352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018873.958314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59cb800 x1818247335604352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335604352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335604352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018873.958353:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335604480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018873.958368:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958374:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59c9180 x1818247335604480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335604480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335604480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018873.958405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335604608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018873.958418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59cbb80 x1818247335604608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335604608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958446:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335604608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018873.958482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018873.958495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59ca680 x1818247335604736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958513:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335604736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958520:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018873.958534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018873.958546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59cad80 x1818247335604864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958562:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335604864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018873.958580:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018873.958626:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018873.958635:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018873/real 1734018873] req@ffff8800b59c9c00 x1818247335604992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018873.958646:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335604992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018873.958654:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018873.958657:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018873.958660:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335604992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018874.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018874.717963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018875.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018875.719946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018876.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018876.721956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018877.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018877.723991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018878.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018878.725987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018878.965804:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.965815:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018878.965820:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.965831:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.965841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.965850:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018878.965888:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335605120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018878.965923:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.965927:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018878.965929:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.965933:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.965936:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.965940:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018878.965949:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018878.965961:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335605248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018878.965970:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.965973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018878.965975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.965979:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.965981:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.965985:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018878.965990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018878.965998:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335605376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018878.966007:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.966010:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018878.966012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.966015:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.966018:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.966021:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018878.966025:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018878.966033:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335605504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018878.966043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.966046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018878.966047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.966051:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.966053:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.966057:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018878.966061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018878.966065:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335605120:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018878.966068:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335605632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018878.966077:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.966080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018878.966082:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.966086:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.966088:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.966091:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018878.966096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018878.966102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335605760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018878.966110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018878.966113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018878.966114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018878.966117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3143 00000100:00080000:0.0:1734018878.966120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018878.966123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018878.966126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018878.966135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335605888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018878.966159:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59c8700 x1818247335605120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335605120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966228:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966233:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335605120:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018878.966241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335605248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018878.966261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59c9f80 x1818247335605248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335605248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335605248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018878.966300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335605376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018878.966315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59ca300 x1818247335605376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335605376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335605376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018878.966378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335605504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018878.966392:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966398:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59cb480 x1818247335605504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966408:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335605504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335605504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018878.966428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335605632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018878.966441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59c8e00 x1818247335605632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335605632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335605632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018878.966478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335605760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018878.966490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59c8a80 x1818247335605760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335605760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335605760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018878.966572:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335605888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018878.966590:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018878.966597:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018878/real 1734018878] req@ffff8800b59c8000 x1818247335605888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018878.966609:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335605888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018878.966617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018878.966619:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018878.966622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335605888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018879.213901:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018879.727951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018880.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018880.729968:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018881.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018881.731944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018882.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018882.733969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018883.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018883.735957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018883.973838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.973848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018883.973853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.973865:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.973875:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018883.973882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018883.973917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335606016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018883.973953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.973957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018883.973959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.973964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.973967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018883.973971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018883.973980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018883.973991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335606144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018883.974001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.974004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018883.974005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.974009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.974012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018883.974015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018883.974020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018883.974029:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335606272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018883.974038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.974044:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018883.974045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.974049:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.974052:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018883.974055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018883.974059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018883.974067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335606400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018883.974076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.974079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018883.974081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.974085:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.974087:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018883.974090:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335606016:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018883.974091:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018883.974095:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018883.974102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335606528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018883.974110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.974112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018883.974114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.974117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.974120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018883.974124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018883.974128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018883.974136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335606656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018883.974144:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018883.974147:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018883.974149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018883.974152:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3148 00000100:00080000:0.0:1734018883.974154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018883.974157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018883.974161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018883.974168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335606784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018883.974188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59cb100 x1818247335606016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335606016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335606016:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018883.974276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335606144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018883.974294:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974301:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59caa00 x1818247335606144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974313:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335606144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974324:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974328:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335606144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018883.974334:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335606272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018883.974349:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59c8000 x1818247335606272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335606272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335606272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018883.974413:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335606400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018883.974475:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59c8a80 x1818247335606400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335606400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335606400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018883.974515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335606528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018883.974530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59c8e00 x1818247335606528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974547:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335606528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974555:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335606528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018883.974569:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335606656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018883.974581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59cb480 x1818247335606656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974597:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335606656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974605:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974608:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974612:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335606656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018883.974617:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335606784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018883.974630:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018883.974636:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018883/real 1734018883] req@ffff8800b59ca300 x1818247335606784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018883.974645:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335606784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018883.974653:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018883.974655:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018883.974659:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335606784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018884.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018884.737995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018885.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018885.739949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018886.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018886.741973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018887.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018887.743997:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018888.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018888.745961:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018888.981832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.981841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018888.981845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.981856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.981865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018888.981873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018888.981911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335606912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018888.981954:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.981959:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018888.981961:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.981965:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.981969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018888.981972:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018888.981982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018888.981993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335607040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018888.982002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.982006:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018888.982007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.982011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.982013:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018888.982016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018888.982021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018888.982028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335607168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018888.982037:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.982040:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018888.982041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.982045:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.982048:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018888.982052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018888.982056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018888.982064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335607296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018888.982072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.982075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018888.982077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.982080:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.982083:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018888.982086:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018888.982090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018888.982097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335607424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018888.982106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.982109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018888.982110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.982114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.982117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018888.982117:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335606912:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018888.982120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018888.982124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018888.982133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335607552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018888.982141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018888.982145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018888.982147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018888.982150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3153 00000100:00080000:0.0:1734018888.982153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018888.982156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018888.982160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018888.982168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335607680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018888.982216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982248:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59c9f80 x1818247335606912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335606912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982284:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335606912:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018888.982301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018888.982320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59c8700 x1818247335607040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335607040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982398:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018888.982405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018888.982423:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982430:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59c9c00 x1818247335607168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335607168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018888.982462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018888.982476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59cad80 x1818247335607296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335607296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982499:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018888.982540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335607424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018888.982554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982561:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59ca680 x1818247335607424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982571:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335607424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982579:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982582:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335607424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018888.982591:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335607552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018888.982604:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982610:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59cbb80 x1818247335607552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335607552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982628:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982630:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982634:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335607552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018888.982640:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335607680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018888.982652:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018888.982657:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018888/real 1734018888] req@ffff8800b59c9180 x1818247335607680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018888.982667:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335607680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018888.982674:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018888.982677:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018888.982679:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335607680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018889.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018889.747964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018890.213947:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018890.749944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018891.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018891.751963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018892.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018892.753967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018893.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018893.755986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018893.989837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.989847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018893.989851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.989862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00080000:0.0:1734018893.989872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.989880:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018893.989917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335607808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018893.989950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.989954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018893.989956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.989961:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00080000:0.0:1734018893.989964:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.989967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018893.989977:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018893.989988:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335607936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018893.989999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.990002:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018893.990004:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.990007:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00080000:0.0:1734018893.990010:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.990013:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018893.990017:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018893.990026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335608064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018893.990035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.990038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018893.990039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.990043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00080000:0.0:1734018893.990045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.990049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018893.990053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018893.990054:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335607808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018893.990060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335608192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018893.990069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.990072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018893.990074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.990078:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00080000:0.0:1734018893.990080:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.990084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018893.990088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018893.990095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335608320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018893.990103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.990106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018893.990107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.990111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00080000:0.0:1734018893.990113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.990116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018893.990120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018893.990120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018893.990128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335608448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018893.990136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018893.990139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018893.990140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018893.990144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3158 00000100:00100000:3.0:1734018893.990144:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59cb800 x1818247335607808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018893.990147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018893.990150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018893.990153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018893.990161:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335608576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018893.990164:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335607808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990175:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990178:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990183:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335607808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018893.990191:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018893.990206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018893.990213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59c9500 x1818247335607936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018893.990224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335607936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990234:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335607936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018893.990244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018893.990301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018893.990309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59c9180 x1818247335608064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018893.990320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335608064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018893.990359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018893.990372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018893.990379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59cbb80 x1818247335608192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018893.990390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335608192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018893.990407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335608320:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018893.990419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018893.990425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59ca680 x1818247335608320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018893.990435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335608320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335608320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018893.990454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335608448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018893.990465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018893.990471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59cad80 x1818247335608448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018893.990480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335608448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990491:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335608448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018893.990499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335608576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018893.990510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018893.990516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018893/real 1734018893] req@ffff8800b59c9c00 x1818247335608576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018893.990526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335608576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018893.990532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018893.990534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018893.990537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335608576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018894.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018894.757982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018895.213940:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018895.759949:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018896.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018896.761953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018897.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018897.763954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018898.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018898.765966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018898.997847:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.997857:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018898.997861:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.997871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.997880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.997887:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018898.997923:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335608704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018898.997957:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.997961:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018898.997963:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.997967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.997971:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.997974:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018898.997984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018898.997994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335608832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018898.998003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.998007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018898.998008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.998013:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.998015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.998019:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018898.998023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018898.998031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335608960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018898.998040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.998042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018898.998044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.998047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.998050:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.998053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018898.998057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018898.998065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335609088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018898.998067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335608704:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018898.998074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.998077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018898.998079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.998082:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.998084:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.998088:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018898.998092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018898.998099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335609216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018898.998108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.998111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018898.998112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.998116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.998118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.998122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018898.998125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018898.998132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335609344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018898.998135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018898.998140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018898.998143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018898.998144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018898.998147:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3163 00000100:00080000:0.0:1734018898.998150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018898.998153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018898.998157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018898.998158:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59c8700 x1818247335608704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734018898.998202:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335609472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018898.998212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335608704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998227:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335608704:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018898.998240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018898.998260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018898.998268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59c9f80 x1818247335608832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018898.998279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335608832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018898.998299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018898.998312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018898.998318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59ca300 x1818247335608960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018898.998329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335608960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998337:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335608960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018898.998368:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018898.998380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018898.998386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59cb480 x1818247335609088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018898.998396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335609088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018898.998414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335609216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018898.998425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018898.998431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59c8e00 x1818247335609216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018898.998440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335609216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335609216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018898.998459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335609344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018898.998470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018898.998476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59c8a80 x1818247335609344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018898.998485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335609344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335609344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018898.998503:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335609472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018898.998514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018898.998520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018898/real 1734018898] req@ffff8800b59c8000 x1818247335609472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018898.998530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335609472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018898.998537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018898.998539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018898.998542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335609472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018899.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018899.767954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018900.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018900.769938:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018901.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018901.771961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018902.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018902.773948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018903.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018903.775906:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018904.005831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.005845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018904.005851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018904.005866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.005879:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.005888:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018904.005938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335609600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018904.005977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.005982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018904.005984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018904.005990:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.005993:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.005997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018904.006008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018904.006021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335609728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018904.006031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.006035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018904.006036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018904.006040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.006043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.006046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018904.006051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018904.006062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335609856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018904.006072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.006076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018904.006077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018904.006109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.006113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.006117:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018904.006121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018904.006131:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335609984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018904.006142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.006145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018904.006147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018904.006151:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335609600:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018904.006152:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.006154:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.006158:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018904.006163:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018904.006171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335610112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018904.006180:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.006183:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018904.006184:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018904.006188:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.006191:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.006195:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018904.006199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018904.006207:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335610240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018904.006215:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018904.006218:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018904.006219:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018904.006223:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3168 00000100:00080000:0.0:1734018904.006226:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018904.006229:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018904.006233:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018904.006241:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335610368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018904.006259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018904.006282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018904.006309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018904.006327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335610112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018904.006343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335610240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018904.006358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335610368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018904.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018904.778031:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018905.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018905.779939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018906.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018906.781994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018907.012011:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018907.012097:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018907.012119:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018907.012156:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018907.012166:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018907.012173:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018907.012181:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018907.012189:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018907.012196:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018907.012276:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59c8a80 x1818247335609984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335609984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012332:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018907.012357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59c8e00 x1818247335610112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335610112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012399:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335610112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018907.012419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59cb480 x1818247335610240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335610240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335610240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018907.012462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59ca300 x1818247335610368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335610368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335610368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018907.012509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59caa00 x1818247335609600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335609600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335609600:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018907.012581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59cb100 x1818247335609728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012601:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335609728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012618:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012624:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018907.012634:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018904/real 1734018907] req@ffff8800b59c8000 x1818247335609856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018907.012650:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335609856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018907.012662:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018907.012665:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018907.012670:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335609856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018907.213909:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018907.784104:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018908.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018908.786050:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018909.013839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.013850:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018909.013856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.013869:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.013881:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.013893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018909.013942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335610496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018909.013987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.013992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018909.013994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.014036:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.014041:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.014045:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018909.014057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018909.014070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335610624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018909.014081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.014084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018909.014086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.014089:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.014092:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.014095:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018909.014100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:3.0:1734018909.014103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335610496:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018909.014109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335610752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018909.014118:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.014121:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018909.014123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.014126:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.014129:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.014133:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018909.014138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018909.014146:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335610880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018909.014154:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.014158:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018909.014159:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.014163:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.014165:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.014169:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018909.014173:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018909.014181:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335611008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018909.014188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018909.014189:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.014192:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018909.014193:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.014197:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.014200:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.014203:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018909.014208:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018909.014216:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335611136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018909.014218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59c9f80 x1818247335610496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018909.014224:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018909.014228:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018909.014230:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018909.014234:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3173 00000100:00080000:0.0:1734018909.014237:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018909.014240:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014240:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335610496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734018909.014244:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734018909.014252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1734018909.014255:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335611264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734018909.014255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014260:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335610496:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018909.014268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335610624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018909.014285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59c8700 x1818247335610624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018909.014304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335610624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018909.014312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018909.014314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335610624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018909.014348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335610752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018909.014363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59c9c00 x1818247335610752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018909.014379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335610752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018909.014395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018909.014398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335610752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018909.014407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335610880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018909.014418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59cad80 x1818247335610880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018909.014434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335610880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018909.014442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018909.014444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335610880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018909.014454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018909.014466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59ca680 x1818247335611008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018909.014482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335611008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018909.014489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018909.014492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018909.014502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018909.014515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59cbb80 x1818247335611136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018909.014531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335611136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018909.014538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018909.014540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018909.014548:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018909.014561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018909.014567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018909/real 1734018909] req@ffff8800b59c9180 x1818247335611264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018909.014576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335611264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018909.014583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018909.014585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018909.014589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018909.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018909.787949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018910.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018910.790018:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018911.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018911.792009:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018912.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018912.793975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018913.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018913.796018:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018914.021820:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.021832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018914.021837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.021847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.021857:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018914.021866:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018914.021902:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335611392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018914.021974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.021978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018914.021980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.021985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.021988:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018914.021992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018914.022002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018914.022014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335611520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018914.022025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.022028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018914.022029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.022033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.022035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018914.022039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018914.022043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018914.022053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335611648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018914.022062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.022065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018914.022066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.022070:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.022072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018914.022076:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018914.022079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018914.022087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335611776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018914.022097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.022100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018914.022101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.022105:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.022108:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018914.022111:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335611392:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018914.022112:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018914.022116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018914.022124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335611904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018914.022132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.022135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018914.022136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.022139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.022142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018914.022145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018914.022149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018914.022156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335612032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018914.022164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018914.022166:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018914.022168:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018914.022171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3178 00000100:00080000:0.0:1734018914.022173:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018914.022176:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018914.022180:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018914.022187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335612160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018914.022205:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022236:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59c9500 x1818247335611392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335611392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335611392:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018914.022289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335611520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018914.022307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022313:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59cb800 x1818247335611520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335611520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022333:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022336:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335611520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018914.022346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335611648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018914.022361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59c9180 x1818247335611648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335611648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335611648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018914.022421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335611776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018914.022434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59cbb80 x1818247335611776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335611776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335611776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018914.022471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018914.022484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59ca680 x1818247335611904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335611904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335611904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018914.022520:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018914.022533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59cad80 x1818247335612032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335612032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022557:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018914.022567:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018914.022580:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018914.022587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018914/real 1734018914] req@ffff8800b59c9c00 x1818247335612160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018914.022596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335612160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018914.022604:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018914.022606:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018914.022609:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018914.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018914.798003:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018915.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018915.799968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018916.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018916.801969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018917.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018917.804040:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018918.213883:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018918.805975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018919.029858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.029868:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018919.029872:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.029883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.029892:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018919.029901:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018919.029936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335612288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018919.029971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.029975:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018919.029977:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.029981:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.029984:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018919.029988:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018919.029998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018919.030009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335612416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018919.030018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.030022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018919.030023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.030027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.030030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018919.030033:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018919.030038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018919.030046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335612544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018919.030055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.030058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018919.030060:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.030063:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.030065:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018919.030069:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018919.030074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018919.030081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335612672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018919.030090:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.030093:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018919.030094:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.030098:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.030100:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018919.030104:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018919.030108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018919.030115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335612800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018919.030125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.030128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018919.030130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.030134:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.030136:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018919.030138:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335612288:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018919.030139:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018919.030144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018919.030151:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335612928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018919.030159:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018919.030162:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018919.030163:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018919.030166:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3183 00000100:00080000:0.0:1734018919.030169:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018919.030172:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018919.030176:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018919.030185:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335613056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018919.030237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030269:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59c8700 x1818247335612288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335612288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335612288:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018919.030322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335612416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018919.030341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59c9f80 x1818247335612416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335612416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030370:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335612416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018919.030380:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335612544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018919.030395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59c8000 x1818247335612544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335612544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030421:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335612544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018919.030433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335612672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018919.030446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59cb100 x1818247335612672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335612672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335612672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018919.030508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018919.030522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59caa00 x1818247335612800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335612800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030548:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018919.030558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018919.030571:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030577:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59ca300 x1818247335612928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030587:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335612928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030595:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030597:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030600:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335612928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018919.030606:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018919.030618:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018919.030624:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018919/real 1734018919] req@ffff8800b59cb480 x1818247335613056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018919.030635:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335613056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018919.030643:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018919.030645:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018919.030648:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018919.213896:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018919.807996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018920.213889:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018920.809995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018921.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018921.811971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018922.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018922.814004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018923.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018923.816013:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018924.037827:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.037837:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018924.037841:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.037853:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.037862:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.037869:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018924.037906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335613184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018924.037948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.037952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018924.037954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.037958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.037961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.037965:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018924.037974:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018924.037985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335613312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018924.037994:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.037997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018924.037999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.038003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.038005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.038009:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018924.038013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018924.038023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335613440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018924.038032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.038035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018924.038036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.038040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.038042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.038047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018924.038051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018924.038059:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335613568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018924.038068:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.038071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018924.038073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.038077:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.038079:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.038083:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018924.038087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018924.038087:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335613184:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018924.038094:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335613696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018924.038102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.038106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018924.038107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.038111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.038113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.038116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018924.038121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018924.038128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335613824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018924.038135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018924.038139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018924.038141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018924.038144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3188 00000100:00080000:0.0:1734018924.038147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018924.038150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018924.038154:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018924.038161:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335613952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018924.038181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59c8e00 x1818247335613184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335613184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335613184:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018924.038266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335613312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018924.038286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59c8a80 x1818247335613312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335613312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335613312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018924.038325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335613440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018924.038340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59cb480 x1818247335613440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335613440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335613440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018924.038409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335613568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018924.038422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59ca300 x1818247335613568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335613568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335613568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018924.038458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018924.038471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59caa00 x1818247335613696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335613696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038496:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018924.038509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018924.038521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59cb100 x1818247335613824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335613824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038545:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018924.038557:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018924.038570:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018924.038576:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018924/real 1734018924] req@ffff8800b59c8000 x1818247335613952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018924.038586:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335613952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018924.038593:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018924.038595:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018924.038598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335613952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018924.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018924.817976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018925.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018925.820003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018926.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018926.821990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018927.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018927.823931:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018928.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018928.825990:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018929.045836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.045848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018929.045852:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.045864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.045874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018929.045882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018929.045918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335614080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018929.045953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.045958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018929.045959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.045964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.045967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018929.045971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018929.045980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018929.045991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335614208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018929.046002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.046005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018929.046007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.046010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.046012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018929.046016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018929.046020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018929.046028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335614336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018929.046036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.046039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018929.046041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.046044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.046046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018929.046050:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018929.046054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018929.046062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335614464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018929.046071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.046074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018929.046075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.046079:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.046082:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018929.046082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335614080:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018929.046086:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018929.046090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018929.046098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335614592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018929.046106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.046108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018929.046110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.046113:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.046116:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018929.046119:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018929.046123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018929.046130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335614720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018929.046138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018929.046141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018929.046142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018929.046145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3193 00000100:00080000:0.0:1734018929.046147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018929.046150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018929.046154:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018929.046163:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335614848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018929.046181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59c9f80 x1818247335614080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335614080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335614080:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018929.046266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335614208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018929.046285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59c8700 x1818247335614208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335614208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335614208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018929.046324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335614336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018929.046338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59c9c00 x1818247335614336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335614336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335614336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018929.046403:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335614464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018929.046417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59cad80 x1818247335614464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046433:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335614464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335614464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018929.046453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335614592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018929.046466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59ca680 x1818247335614592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335614592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335614592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018929.046501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335614720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018929.046512:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59cbb80 x1818247335614720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335614720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335614720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018929.046546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335614848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018929.046558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018929.046564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018929/real 1734018929] req@ffff8800b59c9180 x1818247335614848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018929.046574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335614848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018929.046581:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018929.046583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018929.046586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335614848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018929.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018929.828045:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018930.213890:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018930.829963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018931.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018931.831935:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018932.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018932.833973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018933.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018933.835931:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018934.053821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.053830:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018934.053834:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018934.053844:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.053854:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.053862:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018934.053898:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335614976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018934.053934:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.053938:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018934.053940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018934.053944:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.053947:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.053951:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018934.053961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018934.053971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335615104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018934.053981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.053984:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018934.053986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018934.053990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.053993:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.053996:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018934.054002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018934.054012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335615232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018934.054020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.054024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018934.054026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018934.054029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.054032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.054035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018934.054039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018934.054046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335615360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018934.054055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.054058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018934.054059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018934.054063:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.054065:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.054068:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018934.054072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018934.054079:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335615488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018934.054089:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.054091:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018934.054093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018934.054097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335614976:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018934.054098:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.054100:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.054104:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018934.054108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018934.054116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335615616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018934.054124:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018934.054127:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018934.054128:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018934.054132:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3198 00000100:00080000:0.0:1734018934.054134:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018934.054137:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018934.054141:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018934.054148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335615744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018934.054198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59cb800 x1818247335614976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335614976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054266:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054269:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335614976:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018934.054282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335615104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018934.054301:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59c9500 x1818247335615104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054320:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335615104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335615104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018934.054341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335615232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018934.054355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054361:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59c9180 x1818247335615232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335615232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335615232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018934.054393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335615360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018934.054406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59cbb80 x1818247335615360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335615360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335615360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018934.054469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335615488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018934.054482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59ca680 x1818247335615488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335615488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335615488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018934.054521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335615616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018934.054534:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054540:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59cad80 x1818247335615616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335615616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054558:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054560:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335615616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018934.054570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335615744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018934.054639:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018934.054648:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018934/real 1734018934] req@ffff8800b59c9c00 x1818247335615744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018934.054659:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335615744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018934.054668:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018934.054670:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018934.054673:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335615744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018934.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018934.837988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018935.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018935.839968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018936.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018936.841945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018937.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018937.844022:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018938.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018938.845998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018939.061834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.061845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018939.061849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.061860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.061869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.061877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018939.061914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335615872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018939.061956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.061961:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018939.061962:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.061967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.061970:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.061974:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018939.061983:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018939.061994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335616000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018939.062003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.062007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018939.062008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.062012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.062014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.062018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018939.062022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018939.062030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335616128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018939.062039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.062042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018939.062043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.062047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.062049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.062054:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018939.062058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018939.062066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335616256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018939.062075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.062078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018939.062080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.062083:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.062085:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.062089:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018939.062093:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018939.062097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335615872:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018939.062101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335616384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018939.062108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.062111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018939.062113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.062116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.062118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.062122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018939.062126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018939.062133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335616512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018939.062141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018939.062145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018939.062146:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018939.062150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3203 00000100:00080000:0.0:1734018939.062152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018939.062155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018939.062159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018939.062168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335616640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018939.062193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59c8700 x1818247335615872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335615872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335615872:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018939.062277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018939.062296:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59c9f80 x1818247335616000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335616000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018939.062335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018939.062350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59c8000 x1818247335616128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335616128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018939.062416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018939.062429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59cb100 x1818247335616256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335616256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062459:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018939.062465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335616384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018939.062477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59caa00 x1818247335616384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335616384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335616384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018939.062557:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335616512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018939.062573:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59ca300 x1818247335616512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335616512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062600:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335616512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018939.062611:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335616640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018939.062624:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018939.062630:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018939/real 1734018939] req@ffff8800b59cb480 x1818247335616640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018939.062639:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335616640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018939.062647:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018939.062649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018939.062652:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335616640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018939.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018939.847953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018940.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018940.849964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018941.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018941.851960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018942.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018942.853890:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018943.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018943.855997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018944.069840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.069851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018944.069855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.069865:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.069874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.069881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018944.069918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335616768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018944.069952:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.069956:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018944.069958:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.069963:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.069966:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.069969:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018944.069979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018944.069989:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335616896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018944.070000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.070004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018944.070005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.070009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.070011:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.070014:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018944.070018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018944.070028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335617024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018944.070036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.070040:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018944.070041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.070045:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.070047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.070051:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018944.070055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018944.070063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335617152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018944.070073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.070076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018944.070078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.070081:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.070084:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.070088:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018944.070092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734018944.070097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335616768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018944.070100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335617280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018944.070108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.070111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018944.070112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.070116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.070118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.070121:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018944.070125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018944.070132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335617408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018944.070140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018944.070143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018944.070144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018944.070147:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3208 00000100:00080000:0.0:1734018944.070150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018944.070153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018944.070156:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018944.070164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335617536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018944.070195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070228:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59c8a80 x1818247335616768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335616768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070264:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335616768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018944.070282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018944.070300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070307:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59c8e00 x1818247335616896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070319:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335616896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070330:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335616896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018944.070340:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018944.070354:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070360:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59cb480 x1818247335617024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335617024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018944.070465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018944.070483:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59ca300 x1818247335617152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335617152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018944.070521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335617280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018944.070534:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070540:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59caa00 x1818247335617280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070551:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335617280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070563:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070566:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335617280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018944.070573:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335617408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018944.070586:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070592:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59cb100 x1818247335617408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070602:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335617408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070609:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070612:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070615:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335617408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018944.070620:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335617536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018944.070633:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018944.070640:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018944/real 1734018944] req@ffff8800b59c8000 x1818247335617536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018944.070651:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335617536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018944.070658:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018944.070660:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018944.070663:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335617536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018944.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018944.857998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018945.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018945.859965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018946.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018946.861936:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018947.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018947.863983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018948.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018948.865960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018949.077839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.077849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018949.077854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.077864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.077874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.077881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018949.077917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335617664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018949.077953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.077957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018949.077959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.077964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.077967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.077971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018949.077980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018949.077991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335617792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018949.078001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.078004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018949.078006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.078009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.078012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.078015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018949.078021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018949.078028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335617920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018949.078037:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.078040:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018949.078041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.078045:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.078048:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.078051:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018949.078055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018949.078063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335618048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018949.078071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.078074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018949.078075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.078079:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.078081:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.078084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018949.078088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018949.078095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335618176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018949.078105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.078108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018949.078110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.078114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.078116:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.078120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335617664:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018949.078124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018949.078133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335618304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018949.078141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018949.078143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018949.078145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018949.078148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3213 00000100:00080000:0.0:1734018949.078150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018949.078153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018949.078157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018949.078164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335618432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018949.078224:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078256:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59c9f80 x1818247335617664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335617664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335617664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018949.078308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018949.078360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078369:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59c8700 x1818247335617792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335617792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018949.078402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018949.078418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59c9c00 x1818247335617920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335617920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335617920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018949.078456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018949.078469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59cad80 x1818247335618048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335618048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018949.078535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335618176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018949.078549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59ca680 x1818247335618176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335618176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335618176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018949.078587:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335618304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018949.078600:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078606:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59cbb80 x1818247335618304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078616:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335618304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078627:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078630:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335618304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018949.078636:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335618432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018949.078649:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018949.078655:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018949/real 1734018949] req@ffff8800b59c9180 x1818247335618432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018949.078665:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335618432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018949.078673:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018949.078676:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018949.078679:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335618432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018949.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018949.867996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018950.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018950.869985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018951.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018951.871949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018952.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018952.873950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018953.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018953.875997:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018954.085825:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.085835:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018954.085839:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.085849:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00080000:0.0:1734018954.085858:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.085865:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018954.085898:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335618560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018954.085941:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.085945:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018954.085947:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.085951:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00080000:0.0:1734018954.085955:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.085958:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018954.085968:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018954.085979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335618688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018954.085988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.085992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018954.085993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.085997:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00080000:0.0:1734018954.085999:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.086002:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018954.086006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018954.086017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335618816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018954.086025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.086028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018954.086030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.086033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00080000:0.0:1734018954.086036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.086040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018954.086044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018954.086051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335618944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018954.086060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.086064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018954.086065:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.086069:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00100000:3.0:1734018954.086069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335618560:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018954.086071:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.086075:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018954.086079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018954.086087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335619072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018954.086095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.086098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018954.086099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.086103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00080000:0.0:1734018954.086105:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.086109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018954.086113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018954.086121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335619200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018954.086128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018954.086132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018954.086133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018954.086137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3218 00000100:00080000:0.0:1734018954.086140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018954.086143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018954.086146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018954.086154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335619328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018954.086168:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59c9500 x1818247335618560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335618560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335618560:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018954.086300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018954.086323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59cb800 x1818247335618688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335618688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018954.086364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018954.086379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59c9180 x1818247335618816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335618816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086406:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086408:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018954.086444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018954.086457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59cbb80 x1818247335618944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335618944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335618944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018954.086493:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018954.086506:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086512:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59ca680 x1818247335619072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335619072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086529:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018954.086543:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018954.086555:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086561:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59cad80 x1818247335619200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086571:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335619200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086579:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086582:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018954.086592:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018954.086604:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018954.086610:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018954/real 1734018954] req@ffff8800b59c9c00 x1818247335619328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018954.086620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335619328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018954.086627:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018954.086629:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018954.086632:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018954.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018954.877957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018955.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018955.879995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018956.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018956.881983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018957.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018957.883964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018958.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018958.885982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018959.093815:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.093826:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018959.093831:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018959.093841:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3223 00000100:00080000:0.0:1734018959.093850:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.093858:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018959.093894:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335619456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018959.093929:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.093933:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018959.093935:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018959.093940:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3223 00000100:00080000:0.0:1734018959.093943:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.093946:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018959.093956:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018959.093966:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335619584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018959.093978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.093981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018959.093983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018959.093987:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3223 00000100:00080000:0.0:1734018959.093989:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.093992:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018959.093993:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335619456:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018959.093997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018959.094006:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335619712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018959.094014:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.094017:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018959.094018:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018959.094022:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3223 00000100:00080000:0.0:1734018959.094024:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.094027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018959.094031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018959.094039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335619840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018959.094048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.094051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018959.094052:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018959.094056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3223 00000100:00080000:0.0:1734018959.094059:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.094062:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018959.094066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018959.094074:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335619968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018959.094081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.094084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018959.094086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018959.094089:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3223 00000800:00000100:3.0:1734018959.094089:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734018959.094092:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.094095:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018959.094099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018959.094106:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335620096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018959.094114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018959.094117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018959.094119:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734018959.094120:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59c8700 x1818247335619456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018959.094122:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3223 00000100:00080000:0.0:1734018959.094125:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018959.094128:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018959.094132:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018959.094141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335620224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018959.094144:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335619456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094199:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094203:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094209:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335619456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018959.094241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335619584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018959.094264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018959.094273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59c9f80 x1818247335619584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018959.094286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335619584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335619584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018959.094308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335619712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018959.094323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018959.094329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59c8000 x1818247335619712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018959.094341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335619712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335619712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018959.094361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335619840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018959.094383:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018959.094389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59cb100 x1818247335619840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018959.094399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335619840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094407:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094409:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094412:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335619840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018959.094418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018959.094432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018959.094438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59caa00 x1818247335619968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018959.094448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335619968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094459:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335619968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018959.094469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018959.094482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018959.094488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59ca300 x1818247335620096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018959.094498:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335620096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094508:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018959.094518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018959.094531:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018959.094537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018959/real 1734018959] req@ffff8800b59cb480 x1818247335620224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018959.094547:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335620224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018959.094554:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018959.094556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018959.094559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018959.213877:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018959.888009:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018960.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018960.889964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018961.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018961.891980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018962.213796:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018962.893960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018963.213904:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018963.895952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018964.101763:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.101771:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018964.101776:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.101786:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.101796:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.101803:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018964.101841:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335620352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018964.101874:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.101878:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018964.101880:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.101884:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.101887:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.101891:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018964.101900:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018964.101911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335620480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018964.101921:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.101924:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018964.101925:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.101930:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.101933:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.101936:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018964.101940:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018964.101949:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335620608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018964.101958:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.101961:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018964.101963:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.101966:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.101969:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.101972:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018964.101976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018964.101984:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335620736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018964.101994:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.101997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018964.101998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.102002:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.102005:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.102008:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102008:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335620352:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734018964.102012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018964.102020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335620864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018964.102029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.102032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018964.102033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.102037:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.102039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.102042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018964.102046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018964.102054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335620992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018964.102100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018964.102103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018964.102104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018964.102108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3228 00000100:00080000:0.0:1734018964.102111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018964.102114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018964.102118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018964.102127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335621120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018964.102137:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102161:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59c8e00 x1818247335620352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102175:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335620352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102184:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102186:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102190:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335620352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018964.102195:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335620480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018964.102210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59c8a80 x1818247335620480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335620480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102230:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335620480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018964.102234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335620608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018964.102244:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102248:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59cb480 x1818247335620608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335620608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335620608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018964.102289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335620736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018964.102298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102301:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59ca300 x1818247335620736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102307:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335620736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335620736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018964.102320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018964.102328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59caa00 x1818247335620864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102356:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335620864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102371:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018964.102379:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018964.102393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59cb100 x1818247335620992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335620992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335620992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018964.102430:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335621120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018964.102444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018964.102450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018964/real 1734018964] req@ffff8800b59c8000 x1818247335621120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018964.102460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335621120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018964.102468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018964.102470:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018964.102474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335621120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018964.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018964.897991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018965.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018965.899970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018966.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018966.901979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018967.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018967.903952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018968.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018968.906057:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018969.109857:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.109866:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018969.109871:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.109881:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.109891:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018969.109899:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018969.109937:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335621248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018969.109972:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.110001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018969.110003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.110009:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.110013:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018969.110016:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018969.110027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018969.110039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335621376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018969.110050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.110053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018969.110055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.110058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.110061:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018969.110064:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018969.110068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018969.110076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335621504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018969.110086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.110089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018969.110091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.110095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.110098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018969.110099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335621248:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018969.110102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018969.110106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018969.110114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335621632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018969.110122:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.110125:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018969.110127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.110130:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.110132:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018969.110136:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018969.110140:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018969.110147:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335621760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018969.110155:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.110158:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018969.110159:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.110162:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.110165:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018969.110169:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018969.110173:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018969.110180:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335621888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018969.110189:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018969.110192:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018969.110194:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018969.110197:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3233 00000100:00080000:0.0:1734018969.110199:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018969.110202:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018969.110206:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018969.110215:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335622016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018969.110225:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335621376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018969.110260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335621504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018969.110278:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335621632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018969.110297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335621760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018969.110315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335621888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018969.110331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018969.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018969.908030:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018970.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018970.910041:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018971.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018971.912003:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734018972.116022:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734018972.116067:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734018972.116088:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018972.116125:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018972.116134:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018972.116142:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018972.116149:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018972.116155:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734018972.116163:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734018972.116255:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59c9c00 x1818247335621504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335621504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335621504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018972.116329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59cad80 x1818247335621632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335621632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335621632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018972.116366:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59ca680 x1818247335621760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335621760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116386:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335621760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018972.116401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59cbb80 x1818247335621888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335621888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335621888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018972.116465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59c9180 x1818247335622016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335622016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018972.116527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59c9f80 x1818247335621248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116547:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335621248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335621248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018972.116590:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018969/real 1734018972] req@ffff8800b59c8700 x1818247335621376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018972.116610:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335621376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018972.116624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018972.116629:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018972.116635:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335621376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734018972.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018972.913995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018973.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018973.916025:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018974.117849:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.117862:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018974.117867:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.117877:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00080000:0.0:1734018974.117887:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.117920:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018974.117954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335622144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018974.117987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.117991:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018974.117993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.117998:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00080000:0.0:1734018974.118001:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.118005:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018974.118016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018974.118027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335622272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018974.118037:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.118040:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018974.118042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.118045:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00080000:0.0:1734018974.118048:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.118051:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018974.118055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018974.118064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335622400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018974.118073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.118076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018974.118078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.118082:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00080000:0.0:1734018974.118084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.118088:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018974.118092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018974.118100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335622528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018974.118109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.118112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018974.118114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.118118:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00100000:3.0:1734018974.118119:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335622144:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018974.118121:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.118125:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018974.118129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018974.118137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335622656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018974.118144:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.118147:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018974.118149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.118152:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00080000:0.0:1734018974.118154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.118157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018974.118161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018974.118168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335622784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018974.118176:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018974.118179:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018974.118180:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018974.118183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3238 00000100:00080000:0.0:1734018974.118186:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018974.118189:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018974.118193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018974.118201:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335622912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018974.118211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118241:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59cb800 x1818247335622144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118264:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335622144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118287:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335622144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018974.118295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335622272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018974.118315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118322:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59c9500 x1818247335622272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335622272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335622272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018974.118354:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335622400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018974.118369:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59c8700 x1818247335622400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335622400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118396:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335622400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018974.118432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335622528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018974.118446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59c9f80 x1818247335622528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335622528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118474:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335622528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018974.118482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018974.118495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59c9180 x1818247335622656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335622656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118523:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018974.118530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018974.118546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59cbb80 x1818247335622784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335622784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018974.118582:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018974.118594:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018974.118600:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018974/real 1734018974] req@ffff8800b59ca680 x1818247335622912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018974.118610:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335622912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734018999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018974.118617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018974.118620:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018974.118623:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335622912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018974.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018974.918008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018975.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018975.919948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018976.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018976.922045:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018977.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018977.924007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018978.213875:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018978.926002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018979.125869:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.125878:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018979.125882:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.125893:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00080000:0.0:1734018979.125902:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.125911:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018979.125947:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335623040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018979.125980:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.125985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018979.125987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.125991:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00080000:0.0:1734018979.125994:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.125998:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018979.126008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018979.126019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335623168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018979.126029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.126032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018979.126034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.126037:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00080000:0.0:1734018979.126040:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.126043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018979.126049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018979.126056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335623296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018979.126066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.126069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018979.126070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.126074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00080000:0.0:1734018979.126076:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.126084:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018979.126088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018979.126095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335623424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018979.126105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.126108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018979.126110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.126113:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00100000:3.0:1734018979.126114:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335623040:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018979.126115:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.126119:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018979.126123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018979.126130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335623552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018979.126139:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.126142:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018979.126144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.126147:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00080000:0.0:1734018979.126150:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.126153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018979.126158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018979.126165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335623680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018979.126172:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018979.126175:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018979.126177:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018979.126180:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3243 00000100:00080000:0.0:1734018979.126183:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018979.126186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018979.126190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018979.126198:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335623808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018979.126211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126242:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59cad80 x1818247335623040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335623040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126276:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335623040:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018979.126293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335623168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018979.126312:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59c9c00 x1818247335623168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126329:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335623168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335623168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018979.126350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335623296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018979.126365:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59c8000 x1818247335623296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335623296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335623296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018979.126436:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335623424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018979.126450:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126456:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59cb100 x1818247335623424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335623424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335623424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018979.126486:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335623552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018979.126499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59caa00 x1818247335623552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335623552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335623552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018979.126535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335623680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018979.126547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126553:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59ca300 x1818247335623680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335623680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335623680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018979.126582:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335623808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018979.126594:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018979.126600:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018979/real 1734018979] req@ffff8800b59cb480 x1818247335623808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018979.126610:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335623808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018979.126617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018979.126619:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018979.126622:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335623808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018979.213887:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018979.928029:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018980.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018980.929988:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018981.213891:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018981.932003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018982.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018982.934017:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018983.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018983.936001:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018984.133831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.133840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018984.133844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.133855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.133865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.133873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018984.133910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335623936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018984.133953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.133957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018984.133959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.133964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.133967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.133971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018984.133981:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018984.133992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335624064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018984.134001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.134004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018984.134006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.134009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.134012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.134015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018984.134019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018984.134030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335624192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018984.134038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.134041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018984.134043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.134046:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.134049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.134053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018984.134057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018984.134064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335624320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018984.134074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.134077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734018984.134077:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335623936:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018984.134079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.134082:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.134085:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.134088:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018984.134092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018984.134100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335624448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018984.134108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.134111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018984.134112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.134116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.134118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.134121:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018984.134125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018984.134132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335624576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018984.134140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018984.134144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018984.134145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018984.134149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3248 00000100:00080000:0.0:1734018984.134152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018984.134155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018984.134159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018984.134166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335624704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018984.134168:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59c8a80 x1818247335623936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335623936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335623936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018984.134250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018984.134269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134276:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59c8e00 x1818247335624064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134288:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335624064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134296:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134302:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018984.134309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018984.134324:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134330:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59cb480 x1818247335624192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335624192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018984.134395:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018984.134409:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134415:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59ca300 x1818247335624320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335624320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134432:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134435:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018984.134445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335624448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018984.134458:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59caa00 x1818247335624448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134473:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335624448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335624448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018984.134495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335624576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018984.134507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59cb100 x1818247335624576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134523:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335624576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335624576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018984.134542:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335624704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018984.134555:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018984.134560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018984/real 1734018984] req@ffff8800b59c8000 x1818247335624704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018984.134570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335624704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018984.134578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018984.134581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018984.134584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335624704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018984.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018984.937969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018985.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018985.939974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018986.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018986.941989:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018987.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018987.943969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018988.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018988.945981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018989.141839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.141848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018989.141853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.141863:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00080000:0.0:1734018989.141873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.141880:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018989.141916:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335624832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018989.141951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.141956:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018989.141958:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.141962:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00080000:0.0:1734018989.141965:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.141969:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018989.141979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018989.141990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335624960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018989.142001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.142004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018989.142006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.142009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00080000:0.0:1734018989.142012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.142015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018989.142019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018989.142027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335625088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018989.142036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.142039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018989.142040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.142044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00080000:0.0:1734018989.142046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.142050:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018989.142053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018989.142061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335625216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018989.142070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.142073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018989.142075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.142079:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00100000:3.0:1734018989.142080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335624832:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018989.142082:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.142085:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018989.142089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018989.142097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335625344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018989.142105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.142107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018989.142109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.142112:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00080000:0.0:1734018989.142115:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.142118:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018989.142122:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018989.142129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335625472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018989.142137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018989.142140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018989.142141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018989.142145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3253 00000100:00080000:0.0:1734018989.142147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018989.142151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018989.142154:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018989.142163:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335625600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018989.142172:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142202:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59c9c00 x1818247335624832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335624832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335624832:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018989.142255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018989.142274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142281:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59cad80 x1818247335624960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335624960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335624960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018989.142314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018989.142329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142335:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59ca680 x1818247335625088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335625088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142354:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018989.142392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018989.142406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59cbb80 x1818247335625216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335625216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018989.142441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335625344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018989.142454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59c9180 x1818247335625344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335625344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335625344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018989.142489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335625472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018989.142502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142508:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59c9f80 x1818247335625472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142518:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335625472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142530:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335625472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018989.142536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335625600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018989.142549:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018989.142554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018989/real 1734018989] req@ffff8800b59c8700 x1818247335625600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018989.142564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335625600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018989.142571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018989.142573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018989.142576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335625600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734018989.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018989.947954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018990.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018990.949978:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018991.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018991.951981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018992.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018992.953942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734018993.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018993.955963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018994.149788:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.149797:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018994.149801:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.149812:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00080000:0.0:1734018994.149821:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.149829:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018994.149864:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335625728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018994.149899:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.149903:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018994.149905:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.149909:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00080000:0.0:1734018994.149912:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.149916:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018994.149926:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018994.149936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335625856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018994.149946:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.149949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018994.149951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.149955:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00080000:0.0:1734018994.149958:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.149961:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018994.149967:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018994.149978:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335625984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018994.149987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.149990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018994.149992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.149995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00080000:0.0:1734018994.149998:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.150001:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018994.150005:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018994.150013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335626112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018994.150022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.150025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018994.150027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.150031:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00100000:3.0:1734018994.150031:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335625728:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734018994.150033:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.150036:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018994.150040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018994.150048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335626240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018994.150056:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.150059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018994.150062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.150065:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00080000:0.0:1734018994.150068:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.150071:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018994.150075:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018994.150083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335626368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018994.150091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018994.150094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018994.150095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018994.150098:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3258 00000100:00080000:0.0:1734018994.150101:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018994.150104:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018994.150108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018994.150115:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335626496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734018994.150125:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150156:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59c9500 x1818247335625728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150178:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335625728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150191:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150200:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335625728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018994.150208:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018994.150227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150234:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59cb800 x1818247335625856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150246:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335625856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150260:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018994.150266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018994.150281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59c8700 x1818247335625984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335625984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335625984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018994.150344:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335626112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018994.150358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59c9f80 x1818247335626112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335626112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335626112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018994.150394:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335626240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018994.150407:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59c9180 x1818247335626240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335626240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150436:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335626240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018994.150443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335626368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018994.150456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59cbb80 x1818247335626368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335626368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335626368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018994.150489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335626496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018994.150502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018994.150508:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018994/real 1734018994] req@ffff8800b59ca680 x1818247335626496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018994.150517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335626496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018994.150525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018994.150527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018994.150530:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335626496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018994.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018994.957966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018995.213790:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018995.959972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018996.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018996.961993:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018997.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018997.963965:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734018998.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734018998.965946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734018999.157834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.157842:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734018999.157847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.157857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.157867:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018999.157875:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734018999.157911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335626624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018999.157944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.157948:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018999.157950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.157954:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.157957:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018999.157961:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018999.157971:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018999.157982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335626752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018999.157991:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.157995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018999.157996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.158000:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.158002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018999.158005:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018999.158010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734018999.158017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335626880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018999.158026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.158031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018999.158033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.158036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.158039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018999.158043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018999.158047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734018999.158047:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335626624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734018999.158054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335627008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018999.158063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.158066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734018999.158068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.158071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.158073:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018999.158077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018999.158080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734018999.158087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335627136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018999.158095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.158098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734018999.158099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.158103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.158105:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734018999.158109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018999.158113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734018999.158118:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734018999.158121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335627264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734018999.158129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734018999.158132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734018999.158134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734018999.158137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3263 00000100:00080000:0.0:1734018999.158139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734018999.158140:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59cad80 x1818247335626624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734018999.158143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734018999.158146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734018999.158155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335627392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734018999.158160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335626624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158171:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158174:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158179:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335626624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734018999.158186:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335626752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018999.158203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018999.158210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59c9c00 x1818247335626752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018999.158220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335626752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335626752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018999.158240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335626880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018999.158253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018999.158259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59c8000 x1818247335626880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018999.158271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335626880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335626880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018999.158309:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018999.158321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018999.158328:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59cb100 x1818247335627008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018999.158337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335627008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158347:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018999.158355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335627136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734018999.158366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018999.158372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59caa00 x1818247335627136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018999.158382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335627136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335627136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734018999.158401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335627264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018999.158412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018999.158418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59ca300 x1818247335627264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018999.158428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335627264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158435:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158440:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335627264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734018999.158445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335627392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734018999.158456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734018999.158462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734018999/real 1734018999] req@ffff8800b59cb480 x1818247335627392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734018999.158523:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335627392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734018999.158530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734018999.158532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734018999.158535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335627392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734018999.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734018999.967968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019000.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019000.969966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019001.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019001.971933:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019002.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019002.973971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019003.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019003.975978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019004.165823:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.165831:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019004.165836:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.165846:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.165855:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019004.165863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019004.165897:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335627520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019004.165938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.165942:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019004.165943:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.165948:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.165951:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019004.165955:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019004.165965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019004.165976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335627648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019004.165985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.165988:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019004.165990:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.165993:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.165996:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019004.165999:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019004.166004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019004.166013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335627776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019004.166021:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.166024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019004.166025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.166029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.166031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019004.166036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019004.166040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019004.166040:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335627520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019004.166048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335627904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019004.166056:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.166059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019004.166061:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.166065:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.166067:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019004.166070:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019004.166074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019004.166081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335628032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019004.166089:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.166092:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019004.166093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.166097:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.166099:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019004.166103:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019004.166107:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019004.166110:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019004.166114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335628160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019004.166122:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019004.166126:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019004.166127:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019004.166131:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3268 00000100:00080000:0.0:1734019004.166133:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019004.166134:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59c8e00 x1818247335627520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019004.166137:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019004.166141:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019004.166148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335628288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019004.166154:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335627520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166164:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166168:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335627520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019004.166179:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019004.166195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019004.166202:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59c8a80 x1818247335627648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019004.166213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335627648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166221:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166227:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019004.166233:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019004.166246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019004.166252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59cb480 x1818247335627776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019004.166263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335627776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019004.166303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019004.166315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019004.166321:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59ca300 x1818247335627904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019004.166331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335627904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166340:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335627904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019004.166349:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019004.166360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019004.166365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59caa00 x1818247335628032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019004.166375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335628032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166433:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019004.166443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019004.166459:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019004.166467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59cb100 x1818247335628160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019004.166478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335628160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019004.166495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019004.166507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019004.166513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019004/real 1734019004] req@ffff8800b59c8000 x1818247335628288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019004.166522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335628288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019004.166530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019004.166532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019004.166536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019004.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019004.977968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019005.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019005.979967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019006.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019006.981934:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019007.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019007.983900:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019008.213933:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019008.985927:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019009.173846:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.173854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019009.173859:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.173869:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.173881:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.173889:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019009.173925:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335628416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019009.173959:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.173964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019009.173966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.173970:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.173973:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.173978:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019009.173988:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019009.173999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335628544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019009.174009:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.174012:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019009.174014:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.174017:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.174020:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.174023:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019009.174027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019009.174035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335628672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019009.174043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.174046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019009.174047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.174051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.174053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.174056:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019009.174060:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019009.174068:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335628800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019009.174073:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335628416:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019009.174076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.174080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019009.174081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.174085:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.174087:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.174090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019009.174094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019009.174101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335628928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019009.174109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.174112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019009.174114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.174117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.174120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.174123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019009.174127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019009.174135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335629056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019009.174141:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019009.174143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019009.174146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019009.174147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019009.174151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3273 00000100:00080000:0.0:1734019009.174153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019009.174157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019009.174161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019009.174165:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59c9c00 x1818247335628416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019009.174169:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335629184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019009.174186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335628416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174196:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174199:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174204:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335628416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019009.174211:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335628544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019009.174227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019009.174234:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59cad80 x1818247335628544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019009.174245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335628544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335628544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019009.174265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335628672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019009.174279:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019009.174285:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59ca680 x1818247335628672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019009.174328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335628672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335628672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019009.174369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335628800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019009.174386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019009.174393:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59cbb80 x1818247335628800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019009.174403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335628800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174416:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335628800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019009.174421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019009.174432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019009.174438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59c9180 x1818247335628928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019009.174448:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335628928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174462:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335628928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019009.174467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019009.174478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019009.174484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59c9f80 x1818247335629056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019009.174493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335629056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174501:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019009.174511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019009.174522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019009.174528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019009/real 1734019009] req@ffff8800b59c8700 x1818247335629184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019009.174537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335629184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019009.174545:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019009.174547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019009.174550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019009.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019009.988000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019010.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019010.990004:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019011.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019011.991979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019012.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019012.993978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019013.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019013.995986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019014.181829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019014.181841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019014.181846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019014.181858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.181869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.181879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019014.181924:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335629312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019014.181963:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019014.181969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019014.181972:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019014.181979:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.181985:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.181992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019014.182007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019014.182026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335629440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019014.182030:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335629312:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019014.182041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019014.182046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019014.182048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019014.182053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.182058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.182063:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019014.182070:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019014.182085:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335629568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019014.182098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019014.182103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019014.182105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019014.182110:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.182114:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.182118:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019014.182122:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734019014.182125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019014.182137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335629696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019014.182149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019014.182154:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59cb800 x1818247335629312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019014.182155:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019014.182158:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019014.182163:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.182168:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.182172:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019014.182178:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019014.182182:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335629312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019014.182191:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335629824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019014.182198:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019014.182202:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335629312:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019014.182250:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019014.182254:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019014.182257:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019014.182263:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.182267:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.182274:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335629440:192.168.201.104@tcp:38:kworker.0 00000008:00100000:0.0:1734019014.182281:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019014.182296:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335629952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019014.182307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019014.182311:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019014.182316:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019014.182318:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019014.182320:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59c9500 x1818247335629440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019014.182325:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3278 00000100:00080000:0.0:1734019014.182329:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019014.182334:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335629440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734019014.182341:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734019014.182351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1734019014.182355:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335630080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019014.182355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335629440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019014.182371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335629568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019014.182396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182407:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59c8700 x1818247335629568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019014.182426:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335629568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019014.182439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019014.182443:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335629568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019014.182455:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335629696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019014.182477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59c9f80 x1818247335629696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019014.182496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335629696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019014.182504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019014.182506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335629696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019014.182514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019014.182527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59c9180 x1818247335629824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019014.182542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335629824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019014.182549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019014.182552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019014.182577:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019014.182590:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182596:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59cbb80 x1818247335629952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019014.182607:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335629952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019014.182615:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019014.182617:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182620:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335629952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019014.182625:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019014.182636:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019014.182642:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019014/real 1734019014] req@ffff8800b59ca680 x1818247335630080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019014.182652:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335630080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019014.182660:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019014.182662:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019014.182665:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019014.213951:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019014.997968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019015.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019015.999967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019016.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019017.001972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019017.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019018.003977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019018.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019019.005991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019019.189852:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.189861:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019019.189865:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.189876:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00080000:0.0:1734019019.189886:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.189893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019019.189928:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335630208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019019.189963:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.189967:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019019.189969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.189973:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00080000:0.0:1734019019.189976:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.189980:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019019.189990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019019.190000:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335630336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019019.190010:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.190013:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019019.190015:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.190018:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00080000:0.0:1734019019.190021:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.190024:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019019.190028:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019019.190036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335630464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019019.190046:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.190049:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019019.190051:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.190054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00080000:0.0:1734019019.190056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.190059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019019.190063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019019.190070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335630592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019019.190080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.190083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019019.190084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.190088:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00100000:3.0:1734019019.190089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335630208:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019019.190090:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.190094:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019019.190098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019019.190106:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335630720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019019.190114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.190116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019019.190118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.190121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00080000:0.0:1734019019.190158:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.190163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019019.190168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019019.190177:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335630848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019019.190186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019019.190189:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019019.190191:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019019.190194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3283 00000100:00080000:0.0:1734019019.190197:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019019.190200:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019019.190204:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019019.190211:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335630976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019019.190223:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190255:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59cad80 x1818247335630208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335630208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190290:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335630208:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019019.190308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335630336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019019.190328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59c9c00 x1818247335630336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335630336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190354:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190357:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190360:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335630336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019019.190367:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335630464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019019.190382:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190388:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59c8000 x1818247335630464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335630464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190411:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190414:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335630464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019019.190448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335630592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019019.190461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59cb100 x1818247335630592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335630592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335630592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019019.190497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019019.190510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59caa00 x1818247335630720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335630720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019019.190547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019019.190559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59ca300 x1818247335630848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335630848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019019.190594:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019019.190606:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019019.190612:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019019/real 1734019019] req@ffff8800b59cb480 x1818247335630976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019019.190621:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335630976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019019.190629:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019019.190631:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019019.190634:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335630976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019019.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019020.008008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019020.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019021.009934:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019021.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019022.011984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019022.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019023.013954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019023.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019024.015976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019024.197778:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.197786:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019024.197790:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.197800:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00080000:0.0:1734019024.197810:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.197816:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019024.197853:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335631104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019024.197879:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.197882:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019024.197883:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.197886:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00080000:0.0:1734019024.197889:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.197891:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019024.197899:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019024.197907:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335631232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019024.197912:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.197914:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019024.197915:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.197918:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00080000:0.0:1734019024.197920:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.197923:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019024.197925:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019024.197932:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335631360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019024.197937:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.197939:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019024.197940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.197943:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00080000:0.0:1734019024.197945:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.197947:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019024.197950:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019024.197956:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335631488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019024.197961:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.197963:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019024.197964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.197967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00100000:3.0:1734019024.197967:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335631104:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019024.197969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.197971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019024.197974:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019024.197979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335631616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019024.197984:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.197986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019024.197987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.197989:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00080000:0.0:1734019024.197990:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.197993:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019024.197996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019024.198020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335631744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019024.198030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019024.198033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019024.198034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019024.198038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3288 00000100:00080000:0.0:1734019024.198072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019024.198091:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019024.198095:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019024.198100:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019024.198102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335631872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019024.198121:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59c8a80 x1818247335631104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198134:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335631104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198142:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198144:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198147:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335631104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019024.198152:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335631232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019024.198165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019024.198170:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59c8e00 x1818247335631232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335631232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198181:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198183:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335631232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019024.198188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335631360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019024.198198:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019024.198201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59cb480 x1818247335631360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335631360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198214:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198215:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198217:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335631360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019024.198238:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335631488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019024.198246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019024.198250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59ca300 x1818247335631488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198255:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335631488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335631488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019024.198267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335631616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019024.198274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019024.198277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59caa00 x1818247335631616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335631616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198289:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335631616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019024.198295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335631744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019024.198302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019024.198305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59cb100 x1818247335631744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335631744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335631744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019024.198321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335631872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019024.198329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019024.198332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019024/real 1734019024] req@ffff8800b59c8000 x1818247335631872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019024.198338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335631872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019024.198342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019024.198343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019024.198345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335631872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019024.213759:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019025.018005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019025.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019026.019985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019026.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019027.021950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019027.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019028.024019:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019028.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019029.026006:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019029.205830:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.205840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019029.205844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.205854:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.205864:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.205872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019029.205907:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335632000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019029.205942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.205947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019029.205949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.205990:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.205994:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.205997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019029.206008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019029.206020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335632128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019029.206030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.206034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019029.206036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.206039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.206041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.206045:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019029.206049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019029.206057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335632256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019029.206067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.206070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734019029.206070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335632000:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019029.206071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.206075:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.206077:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.206081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019029.206085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019029.206093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335632384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019029.206102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.206106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019029.206107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.206111:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.206113:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.206117:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019029.206121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019029.206128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335632512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019029.206135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.206138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019029.206140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.206143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.206146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.206149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019029.206153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019029.206160:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335632640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019029.206162:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019029.206168:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019029.206171:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019029.206172:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019029.206175:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3293 00000100:00080000:0.0:1734019029.206179:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019029.206182:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019029.206186:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019029.206191:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59c9c00 x1818247335632000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019029.206195:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335632768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019029.206214:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335632000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206227:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206237:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335632000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019029.206245:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335632128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019029.206266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019029.206273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59cad80 x1818247335632128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019029.206284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335632128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206293:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206296:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335632128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019029.206329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335632256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019029.206344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019029.206351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59ca680 x1818247335632256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019029.206362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335632256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335632256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019029.206382:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335632384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019029.206394:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019029.206400:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59cbb80 x1818247335632384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019029.206410:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335632384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206417:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335632384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019029.206429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335632512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019029.206443:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019029.206449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59c9180 x1818247335632512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019029.206459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335632512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206466:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335632512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019029.206479:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335632640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019029.206492:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019029.206498:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59c9f80 x1818247335632640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019029.206508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335632640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206518:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335632640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019029.206528:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335632768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019029.206541:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019029.206547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019029/real 1734019029] req@ffff8800b59c8700 x1818247335632768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019029.206557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335632768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019029.206564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019029.206566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019029.206569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335632768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019029.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019030.027982:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019030.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019031.030047:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019031.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019032.032011:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019032.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019033.033973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019033.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019034.036019:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019034.213843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.213851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019034.213856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.213866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.213903:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.213911:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019034.213943:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:0.0:1734019034.213946:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335632896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019034.213977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.213981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019034.213983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.213987:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.213990:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.213994:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019034.214003:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019034.214014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335633024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019034.214024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.214027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019034.214029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.214032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.214035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.214038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019034.214042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019034.214052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335633152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019034.214060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.214064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019034.214066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.214069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.214072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.214075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019034.214079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019034.214086:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335633280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019034.214096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.214099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019034.214101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.214105:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.214107:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.214111:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019034.214115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019034.214115:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335632896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019034.214123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335633408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019034.214131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.214134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019034.214135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.214139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.214141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.214144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019034.214148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019034.214156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335633536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019034.214164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019034.214166:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019034.214168:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019034.214171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3298 00000100:00080000:0.0:1734019034.214173:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019034.214176:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019034.214181:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019034.214188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335633664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019034.214241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019034.214265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019034.214294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019034.214312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335633408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019034.214330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335633536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019034.214347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335633664:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019035.038014:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019035.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019036.040030:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019036.213890:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019037.041966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019037.213895:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734019037.220019:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019037.220064:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019037.220085:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019037.220122:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019037.220132:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019037.220140:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019037.220146:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019037.220153:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019037.220160:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019037.220274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59c9f80 x1818247335633280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335633280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019037.220373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59c9180 x1818247335633408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335633408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335633408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019037.220442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59cbb80 x1818247335633536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335633536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335633536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019037.220494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59ca680 x1818247335633664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220513:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335633664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335633664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019037.220569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59c9500 x1818247335632896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335632896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220600:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220603:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335632896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019037.220627:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59cb800 x1818247335633024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220646:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335633024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220659:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220663:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220669:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019037.220681:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019034/real 1734019037] req@ffff8800b59c8700 x1818247335633152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019037.220700:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335633152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019037.220714:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019037.220718:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019037.220951:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633152:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019038.044027:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019038.213969:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019039.046003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019039.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019039.221826:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.221834:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019039.221839:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.221848:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.221858:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.221866:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019039.221902:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335633792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019039.221935:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.221939:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019039.221941:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.221945:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.221948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.221952:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019039.221962:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019039.221972:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335633920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019039.221981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.221985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019039.221986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.221990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.221994:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.221998:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019039.222002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019039.222010:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335634048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019039.222019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.222022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019039.222024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.222027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.222030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.222033:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019039.222037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019039.222044:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335634176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019039.222054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.222057:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019039.222058:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.222062:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.222064:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.222068:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335633792:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019039.222072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019039.222080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335634304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019039.222094:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.222097:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019039.222098:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.222102:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.222104:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.222107:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019039.222111:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019039.222121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335634432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019039.222128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019039.222131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019039.222133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019039.222136:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3303 00000100:00080000:0.0:1734019039.222139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019039.222142:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019039.222146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019039.222152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335634560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019039.222167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59cad80 x1818247335633792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335633792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335633792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019039.222249:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019039.222267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59c9c00 x1818247335633920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335633920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335633920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019039.222306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019039.222320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59c8000 x1818247335634048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335634048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019039.222385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019039.222399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59cb100 x1818247335634176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335634176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222424:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019039.222435:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335634304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019039.222447:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222453:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59caa00 x1818247335634304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222463:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335634304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222476:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335634304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019039.222482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335634432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019039.222498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59ca300 x1818247335634432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335634432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335634432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019039.222534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335634560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019039.222547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019039.222553:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019039/real 1734019039] req@ffff8800b59cb480 x1818247335634560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019039.222563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335634560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019039.222570:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019039.222572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019039.222575:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335634560:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019040.048070:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019040.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019041.050007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019041.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019042.051952:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019042.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019043.053974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019043.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019044.056006:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019044.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019044.229808:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.229816:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019044.229821:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.229831:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.229840:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.229848:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019044.229884:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335634688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019044.229918:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.229922:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019044.229924:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.229929:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.229932:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.229935:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019044.229945:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019044.229955:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335634816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019044.229965:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.229968:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019044.229970:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.229973:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.229976:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.229980:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019044.229984:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019044.229994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335634944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019044.230004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.230007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019044.230009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.230012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.230015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.230018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019044.230022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019044.230031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335635072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019044.230038:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335634688:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019044.230041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.230044:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019044.230045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.230049:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.230051:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.230055:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019044.230059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019044.230066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335635200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019044.230074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.230077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019044.230078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.230082:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.230085:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.230089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019044.230093:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019044.230100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335635328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019044.230107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019044.230110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019044.230111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019044.230115:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3308 00000100:00080000:0.0:1734019044.230118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019044.230121:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019044.230124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019044.230130:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019044.230133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335635456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019044.230161:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59c8e00 x1818247335634688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230183:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335634688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230196:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230200:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230205:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335634688:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019044.230213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019044.230233:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019044.230240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59c8a80 x1818247335634816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335634816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019044.230272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019044.230287:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019044.230293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59cb480 x1818247335634944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335634944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335634944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019044.230358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019044.230372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019044.230378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59ca300 x1818247335635072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335635072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019044.230408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335635200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019044.230421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019044.230427:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59caa00 x1818247335635200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335635200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335635200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019044.230456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335635328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019044.230469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019044.230475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59cb100 x1818247335635328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335635328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335635328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019044.230504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335635456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019044.230517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019044.230523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019044/real 1734019044] req@ffff8800b59c8000 x1818247335635456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019044.230532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335635456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019044.230539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019044.230542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019044.230545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335635456:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019045.057939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019045.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019046.059985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019046.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019047.061984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019047.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019048.063923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019048.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019049.065922:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019049.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019049.237807:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237814:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019049.237817:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237825:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237831:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019049.237837:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019049.237865:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335635584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019049.237887:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237889:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019049.237890:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237892:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237894:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019049.237896:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019049.237903:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019049.237908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335635712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019049.237914:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237915:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019049.237916:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237918:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237920:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019049.237921:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019049.237924:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019049.237928:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335635840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019049.237932:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237933:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019049.237934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237937:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237938:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019049.237940:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019049.237942:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019049.237947:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335635968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019049.237951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019049.237953:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019049.237958:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019049.237960:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019049.237964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335636096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019049.237967:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019049.237970:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237971:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237973:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019049.237974:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019049.237977:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019049.237981:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335636224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019049.237986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019049.237988:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019049.237988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019049.237990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3313 00000100:00080000:0.0:1734019049.237991:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019049.237992:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335635584:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019049.237993:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019049.237995:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019049.238001:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335636352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019049.238056:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238076:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59c9c00 x1818247335635584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238089:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335635584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238096:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238098:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238101:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335635584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019049.238105:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019049.238115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238119:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59cad80 x1818247335635712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238125:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335635712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238130:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238131:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238133:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019049.238136:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019049.238143:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238146:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59c8700 x1818247335635840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238151:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335635840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238156:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238158:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238160:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019049.238163:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019049.238170:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59cb800 x1818247335635968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238178:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335635968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238182:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238184:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335635968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019049.238189:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019049.238196:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238199:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59c9500 x1818247335636096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238204:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335636096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238210:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019049.238235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019049.238242:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59ca680 x1818247335636224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238251:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335636224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019049.238262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019049.238268:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019049.238271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019049/real 1734019049] req@ffff8800b59cbb80 x1818247335636352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019049.238276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335636352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019049.238280:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019049.238281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019049.238283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636352:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019050.067945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019050.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019051.069871:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019051.213760:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019052.071977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019052.213773:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019053.073957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019053.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019054.075972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019054.213924:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019054.245832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.245841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019054.245845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.245855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.245865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.245872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019054.245908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335636480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019054.245943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.245947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019054.245949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.245954:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.245957:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.245960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019054.245970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019054.245981:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335636608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019054.245991:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.245994:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019054.245996:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.245999:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.246002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.246005:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019054.246009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019054.246019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335636736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019054.246028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.246031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019054.246032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.246036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.246038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.246042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019054.246046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019054.246054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335636864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019054.246064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.246067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019054.246069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.246073:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.246075:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.246078:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335636480:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019054.246082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019054.246090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335636992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019054.246098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.246101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019054.246102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.246105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.246108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.246111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019054.246115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019054.246123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335637120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019054.246131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019054.246133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019054.246135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019054.246139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3318 00000100:00080000:0.0:1734019054.246142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019054.246145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019054.246149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019054.246156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335637248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019054.246178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59c9180 x1818247335636480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335636480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246253:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335636480:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019054.246261:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335636608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019054.246279:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246286:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59c9f80 x1818247335636608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335636608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335636608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019054.246318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335636736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019054.246332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59cbb80 x1818247335636736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335636736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335636736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019054.246398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335636864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019054.246411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59ca680 x1818247335636864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246428:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335636864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335636864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019054.246448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019054.246460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59c9500 x1818247335636992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335636992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335636992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019054.246496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335637120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019054.246509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59cb800 x1818247335637120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335637120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246586:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335637120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019054.246596:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335637248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019054.246613:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019054.246621:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019054/real 1734019054] req@ffff8800b59c8700 x1818247335637248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019054.246632:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335637248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019054.246640:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019054.246643:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019054.246646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335637248:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019055.077985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019055.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019056.079979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019056.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019057.081966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019057.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019058.083991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019058.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019059.085998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019059.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019059.253829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.253838:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019059.253842:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.253852:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.253861:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019059.253869:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019059.253905:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335637376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019059.253938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.253942:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019059.253944:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.253949:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.253952:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019059.253956:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019059.253966:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019059.253977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335637504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019059.253986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.253989:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019059.253992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.253996:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.253998:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019059.254002:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019059.254006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019059.254014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335637632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019059.254023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.254025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019059.254027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.254030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.254033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019059.254036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019059.254040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019059.254048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335637760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019059.254057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.254060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019059.254062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.254066:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.254068:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019059.254068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335637376:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019059.254071:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019059.254076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019059.254084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335637888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019059.254092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.254096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019059.254097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.254101:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.254103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019059.254106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019059.254110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019059.254118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335638016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019059.254125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019059.254128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019059.254129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019059.254133:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3323 00000100:00080000:0.0:1734019059.254135:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019059.254138:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019059.254142:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019059.254152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335638144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019059.254161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254191:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59cad80 x1818247335637376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335637376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254225:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335637376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019059.254243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335637504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019059.254261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59c9c00 x1818247335637504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335637504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254291:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335637504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019059.254301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335637632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019059.254316:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254322:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59c8000 x1818247335637632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335637632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254349:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254351:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335637632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019059.254388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335637760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019059.254402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59cb100 x1818247335637760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335637760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254432:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335637760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019059.254438:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335637888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019059.254499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59caa00 x1818247335637888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335637888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335637888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019059.254540:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019059.254553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59ca300 x1818247335638016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335638016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254584:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019059.254589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019059.254603:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019059.254609:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019059/real 1734019059] req@ffff8800b59cb480 x1818247335638144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019059.254618:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335638144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019059.254626:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019059.254628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019059.254631:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638144:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019060.087975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019060.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019061.089980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019061.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019062.091996:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019062.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019063.093959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019063.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019064.096020:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019064.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019064.261824:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.261832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019064.261837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.261847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.261856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.261865:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019064.261901:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335638272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019064.261934:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.261938:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019064.261940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.261945:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.261948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.261951:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019064.261960:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019064.261970:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335638400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019064.261980:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.261984:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019064.261985:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.261989:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.261991:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.261995:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019064.261999:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019064.262009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335638528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019064.262018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.262021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019064.262023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.262027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.262030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.262033:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019064.262037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019064.262044:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335638656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019064.262054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.262057:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019064.262058:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.262062:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.262064:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.262068:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019064.262072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019064.262074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335638272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019064.262080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335638784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019064.262088:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.262090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019064.262092:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.262095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.262098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.262101:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019064.262105:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019064.262112:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335638912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019064.262121:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019064.262124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019064.262126:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019064.262129:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3328 00000100:00080000:0.0:1734019064.262131:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019064.262134:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019064.262138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019064.262145:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335639040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019064.262165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262194:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59c8a80 x1818247335638272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262216:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335638272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335638272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019064.262246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335638400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019064.262265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59c8e00 x1818247335638400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335638400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262298:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335638400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019064.262304:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335638528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019064.262319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59cb480 x1818247335638528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335638528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335638528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019064.262429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335638656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019064.262448:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59ca300 x1818247335638656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335638656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262477:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262480:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335638656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019064.262486:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019064.262499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59caa00 x1818247335638784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335638784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262523:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262527:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262530:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019064.262536:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019064.262548:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59cb100 x1818247335638912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335638912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262572:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262574:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335638912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019064.262584:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019064.262598:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019064.262604:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019064/real 1734019064] req@ffff8800b59c8000 x1818247335639040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019064.262613:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335639040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019064.262621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019064.262623:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019064.262626:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639040:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019065.098000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019065.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019066.099935:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019066.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019067.102000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019067.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019068.103954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019068.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019069.106009:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019069.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019069.269836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.269848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019069.269854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.269867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.269879:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.269889:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019069.269935:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335639168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019069.269974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.269978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019069.269980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.269985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.269989:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.269993:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019069.270006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019069.270019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335639296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019069.270029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.270033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019069.270034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.270038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.270041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.270044:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019069.270049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019069.270058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335639424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019069.270066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.270069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019069.270071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.270074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.270077:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.270081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019069.270085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019069.270086:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335639168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019069.270093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335639552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019069.270102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.270105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019069.270108:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.270112:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.270115:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.270118:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019069.270123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019069.270130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335639680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019069.270138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.270141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019069.270142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.270146:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.270148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.270152:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019069.270156:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019069.270164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335639808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019069.270170:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019069.270172:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019069.270175:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019069.270176:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019069.270180:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3333 00000100:00080000:0.0:1734019069.270183:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019069.270186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019069.270190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019069.270197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59c9c00 x1818247335639168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019069.270199:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335639936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019069.270219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335639168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335639168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019069.270247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335639296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019069.270263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019069.270271:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59cad80 x1818247335639296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019069.270327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335639296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270335:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335639296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019069.270348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335639424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019069.270366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019069.270373:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59c8700 x1818247335639424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019069.270385:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335639424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270393:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270395:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270399:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335639424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019069.270427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335639552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019069.270440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019069.270446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59cb800 x1818247335639552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019069.270456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335639552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335639552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019069.270475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019069.270486:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019069.270492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59c9500 x1818247335639680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019069.270502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335639680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019069.270521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019069.270533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019069.270539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59ca680 x1818247335639808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019069.270549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335639808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019069.270566:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019069.270578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019069.270584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019069/real 1734019069] req@ffff8800b59cbb80 x1818247335639936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019069.270594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335639936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019069.270601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019069.270603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019069.270606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335639936:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019070.108093:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019070.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019071.109971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019071.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019072.111934:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019072.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019073.113911:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019073.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019074.115970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019074.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019074.277818:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.277826:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019074.277830:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.277840:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.277849:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.277857:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019074.277892:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335640064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019074.277927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.277931:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019074.277933:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.277937:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.277940:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.277944:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019074.277954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019074.277964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335640192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019074.277975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.277978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019074.277980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.277983:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.277986:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.277989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019074.277994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019074.278004:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335640320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019074.278013:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.278016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019074.278017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.278021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.278023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.278027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019074.278031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019074.278039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335640448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019074.278048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.278051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019074.278052:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.278056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.278059:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.278062:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278064:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335640064:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019074.278067:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019074.278074:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335640576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019074.278082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.278085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019074.278087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.278090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.278093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.278096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019074.278100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019074.278108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335640704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019074.278116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019074.278119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019074.278120:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019074.278123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3338 00000100:00080000:0.0:1734019074.278126:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019074.278129:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019074.278133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019074.278140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335640832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019074.278158:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59c9f80 x1818247335640064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278238:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335640064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278251:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335640064:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019074.278269:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335640192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019074.278290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59c9180 x1818247335640192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335640192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335640192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019074.278330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335640320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019074.278345:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59cbb80 x1818247335640320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335640320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335640320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019074.278409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335640448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019074.278422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59ca680 x1818247335640448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335640448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278449:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335640448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019074.278458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335640576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019074.278470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59c9500 x1818247335640576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335640576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335640576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019074.278508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335640704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019074.278520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59cb800 x1818247335640704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335640704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335640704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019074.278554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335640832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019074.278567:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019074.278573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019074/real 1734019074] req@ffff8800b59c8700 x1818247335640832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019074.278583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335640832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019074.278591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019074.278593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019074.278597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335640832:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019075.117968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019075.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019076.119975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019076.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019077.121965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019077.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019078.123957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019078.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019079.125963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019079.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:0.0:1734019079.285832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.285840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019079.285845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.285854:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.285865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.285872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019079.285907:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335640960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019079.285948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.285952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019079.285954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.285958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.285961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.285965:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019079.285974:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019079.285985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335641088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019079.285994:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.285997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019079.285999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.286003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.286005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.286009:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019079.286013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019079.286021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335641216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019079.286030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.286033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019079.286034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.286038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.286040:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.286044:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019079.286048:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019079.286056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335641344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019079.286065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.286068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019079.286070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.286073:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.286076:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.286079:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019079.286083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019079.286084:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335640960:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019079.286091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335641472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019079.286099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.286138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019079.286140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.286145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.286148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.286151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019079.286156:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019079.286166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335641600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019079.286175:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019079.286178:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019079.286180:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019079.286183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3343 00000100:00080000:0.0:1734019079.286186:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019079.286189:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019079.286193:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019079.286202:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335641728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019079.286216:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59cad80 x1818247335640960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335640960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335640960:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019079.286299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019079.286318:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59c9c00 x1818247335641088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286337:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335641088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286346:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286352:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019079.286358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019079.286373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59c8000 x1818247335641216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335641216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286402:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019079.286439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019079.286452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286459:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59cb100 x1818247335641344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335641344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019079.286489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335641472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019079.286501:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59caa00 x1818247335641472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286517:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335641472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286525:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335641472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019079.286539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335641600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019079.286551:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286557:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59ca300 x1818247335641600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335641600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335641600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019079.286585:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335641728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019079.286597:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019079.286603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019079/real 1734019079] req@ffff8800b59cb480 x1818247335641728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019079.286612:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335641728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019079.286620:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019079.286622:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019079.286625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335641728:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019080.127967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019080.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019081.129943:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019081.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019082.131946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019082.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019083.133972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019083.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019084.135985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019084.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019084.293947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.293955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019084.293959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.293968:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.293978:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019084.293986:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019084.294062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335641856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019084.294099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.294104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019084.294106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.294111:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.294115:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019084.294118:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019084.294128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019084.294140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335641984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019084.294150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.294153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019084.294155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.294159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.294161:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019084.294164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019084.294169:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019084.294177:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335642112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019084.294186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.294189:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019084.294190:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.294194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.294196:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019084.294199:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019084.294203:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019084.294213:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335642240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019084.294223:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.294226:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019084.294228:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.294232:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.294234:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019084.294236:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335641856:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019084.294238:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019084.294242:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019084.294250:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335642368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019084.294258:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.294260:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019084.294262:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.294265:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.294268:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019084.294271:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019084.294275:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019084.294283:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335642496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019084.294291:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019084.294294:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019084.294296:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019084.294299:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3348 00000100:00080000:2.0:1734019084.294317:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019084.294321:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019084.294325:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019084.294329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019084.294333:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335642624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019084.294358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86df100 x1818247335641856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294381:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335641856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294395:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294404:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335641856:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019084.294412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019084.294431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019084.294438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86dc700 x1818247335641984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335641984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335641984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019084.294469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335642112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019084.294484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019084.294490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86de680 x1818247335642112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335642112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335642112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019084.294544:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335642240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019084.294558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019084.294564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86ddc00 x1818247335642240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335642240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335642240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019084.294594:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335642368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019084.294606:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019084.294612:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86dc000 x1818247335642368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294622:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335642368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294629:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294633:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335642368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019084.294643:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335642496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019084.294655:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019084.294661:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86dd180 x1818247335642496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294671:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335642496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294678:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294680:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294683:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335642496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019084.294797:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335642624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019084.294816:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019084.294824:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019084/real 1734019084] req@ffff8800a86dd500 x1818247335642624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019084.294834:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335642624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019084.294842:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019084.294844:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019084.294848:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335642624:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019085.138051:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019085.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019086.139915:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019086.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019087.141946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019087.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019088.144000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019088.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019089.145986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019089.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019089.301837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.301845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019089.301849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.301859:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.301868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.301876:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019089.301911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335642752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019089.301988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.301993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019089.301995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.302000:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.302004:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.302007:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019089.302020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019089.302032:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335642880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019089.302043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.302046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019089.302048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.302051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.302054:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.302057:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019089.302062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019089.302070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335643008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019089.302079:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.302081:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019089.302083:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.302086:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.302089:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.302092:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019089.302096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019089.302108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335643136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019089.302113:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335642752:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019089.302118:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.302121:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019089.302123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.302127:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.302130:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.302134:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019089.302138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019089.302146:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335643264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019089.302154:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.302157:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019089.302158:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.302162:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.302164:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.302167:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019089.302171:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019089.302178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335643392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019089.302186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019089.302189:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019089.302190:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019089.302193:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3353 00000100:00080000:2.0:1734019089.302197:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019089.302200:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019089.302204:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019089.302207:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019089.302211:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335643520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019089.302237:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86df480 x1818247335642752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335642752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302272:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335642752:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019089.302289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335642880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019089.302307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019089.302314:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86dd880 x1818247335642880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302326:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335642880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302337:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302340:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335642880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019089.302346:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019089.302361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019089.302368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86dca80 x1818247335643008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335643008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019089.302423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019089.302436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019089.302442:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86dd500 x1818247335643136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335643136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019089.302471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335643264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019089.302484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019089.302490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86dd180 x1818247335643264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335643264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302511:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335643264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019089.302520:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335643392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019089.302532:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019089.302538:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86dc000 x1818247335643392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302548:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335643392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335643392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019089.302567:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335643520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019089.302580:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019089.302586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019089/real 1734019089] req@ffff8800a86ddc00 x1818247335643520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019089.302596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335643520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019089.302603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019089.302605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019089.302608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335643520:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019090.147989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019090.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019091.149966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019091.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019092.151961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019092.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019093.153966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019093.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019094.156007:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019094.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019094.309824:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.309832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019094.309837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.309870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.309883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.309891:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019094.309926:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335643648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019094.309959:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.309964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019094.309966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.309970:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.309973:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.309977:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019094.309986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019094.309997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335643776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019094.310006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.310010:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019094.310012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.310016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.310019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.310022:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019094.310026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019094.310034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335643904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019094.310042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.310045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019094.310046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.310050:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.310053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.310056:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019094.310060:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019094.310069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335644032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019094.310079:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.310082:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019094.310083:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.310087:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.310089:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.310093:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019094.310098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019094.310099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335643648:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019094.310106:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335644160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019094.310114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.310117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019094.310119:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.310122:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.310124:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.310127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019094.310131:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019094.310138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335644288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019094.310146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019094.310149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019094.310150:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019094.310153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3358 00000100:00080000:2.0:1734019094.310179:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019094.310182:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019094.310186:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019094.310191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019094.310194:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335644416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019094.310221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86de680 x1818247335643648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335643648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335643648:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019094.310273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019094.310292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019094.310299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86dc700 x1818247335643776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335643776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019094.310331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019094.310345:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019094.310351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86df100 x1818247335643904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335643904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310371:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335643904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019094.310406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019094.310420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019094.310426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86de300 x1818247335644032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335644032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019094.310456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335644160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019094.310469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019094.310475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86dce00 x1818247335644160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335644160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335644160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019094.310504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335644288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019094.310516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019094.310522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86dea00 x1818247335644288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335644288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310539:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335644288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019094.310550:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335644416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019094.310564:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019094.310570:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019094/real 1734019094] req@ffff8800a86dc380 x1818247335644416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019094.310580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335644416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019094.310588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019094.310590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019094.310593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335644416:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019095.158030:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019095.213885:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019096.159987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019096.213873:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019097.161989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019097.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019098.164012:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019098.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019099.165974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019099.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019099.317838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.317849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019099.317854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.317866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00080000:2.0:1734019099.317878:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.317888:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019099.317937:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335644544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019099.317974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.317978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019099.317980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.317985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00080000:2.0:1734019099.317989:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.317992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019099.318003:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019099.318015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335644672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019099.318024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.318028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019099.318029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.318033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00080000:2.0:1734019099.318037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.318041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019099.318046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019099.318054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335644800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019099.318063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.318066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019099.318067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.318071:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00080000:2.0:1734019099.318073:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.318077:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019099.318081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019099.318088:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335644544:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019099.318091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335644928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019099.318100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.318104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019099.318105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.318109:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00080000:2.0:1734019099.318112:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.318116:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019099.318120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019099.318129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335645056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019099.318144:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.318147:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019099.318149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.318153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00080000:2.0:1734019099.318164:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.318168:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019099.318172:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019099.318180:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335645184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019099.318189:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019099.318192:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019099.318194:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019099.318198:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3363 00000100:00100000:3.0:1734019099.318199:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644672:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734019099.318200:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019099.318204:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019099.318208:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019099.318216:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335645312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019099.318221:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019099.318247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019099.318262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019099.318279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019099.318294:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645312:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019100.168000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019100.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019101.170012:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019101.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019102.171957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019102.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734019102.323986:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019102.324045:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019102.324069:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019102.324120:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019102.324135:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019102.324147:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019102.324159:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019102.324170:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019102.324182:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019102.324273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86dc380 x1818247335644928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335644928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324345:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019102.324367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86dea00 x1818247335645056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335645056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324423:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019102.324436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86dce00 x1818247335645184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335645184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019102.324489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86de300 x1818247335645312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335645312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324531:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019102.324542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86df800 x1818247335644544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335644544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324586:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324591:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335644544:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019102.324608:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86dfb80 x1818247335644672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324628:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335644672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324641:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324645:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324651:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019102.324683:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019099/real 1734019102] req@ffff8800a86ddf80 x1818247335644800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019102.324854:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335644800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019102.324868:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019102.324873:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019102.324878:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335644800:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019103.174004:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019103.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019104.175978:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019104.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019104.325805:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.325814:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019104.325818:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.325828:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00080000:2.0:1734019104.325837:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.325845:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019104.325881:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335645440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019104.325914:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.325918:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019104.325920:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.325925:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00080000:2.0:1734019104.325928:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.325931:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019104.325941:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019104.325951:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335645568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019104.325960:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.325964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019104.325965:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.325969:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00080000:2.0:1734019104.325971:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.325975:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019104.325979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019104.325987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335645696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019104.325996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.326000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019104.326001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.326005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00080000:2.0:1734019104.326007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.326010:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019104.326014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019104.326023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335645824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019104.326033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.326036:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019104.326037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.326041:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00100000:3.0:1734019104.326041:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335645440:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019104.326043:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.326047:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019104.326051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019104.326060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335645952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019104.326068:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.326071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019104.326073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.326076:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00080000:2.0:1734019104.326079:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.326082:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019104.326086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019104.326093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335646080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019104.326101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019104.326104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019104.326105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019104.326109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3368 00000100:00080000:2.0:1734019104.326121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019104.326124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019104.326128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019104.326133:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019104.326136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335646208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019104.326163:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86df100 x1818247335645440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326185:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335645440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326199:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326202:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335645440:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019104.326215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335645568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019104.326233:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019104.326240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86dc700 x1818247335645568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335645568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335645568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019104.326272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335645696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019104.326287:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019104.326293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86de680 x1818247335645696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335645696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326321:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335645696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019104.326353:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335645824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019104.326366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019104.326372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86ddc00 x1818247335645824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335645824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326397:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335645824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019104.326403:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019104.326415:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019104.326421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86dc000 x1818247335645952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335645952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335645952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019104.326452:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019104.326464:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019104.326470:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86dd180 x1818247335646080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335646080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019104.326498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019104.326510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019104.326516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019104/real 1734019104] req@ffff8800a86dd500 x1818247335646208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019104.326526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335646208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019104.326533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019104.326535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019104.326538:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646208:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019105.177936:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019105.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019106.180009:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019106.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019107.181952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019107.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019108.183983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019108.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019109.185979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019109.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019109.333810:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.333819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019109.333823:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019109.333833:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.333842:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.333849:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019109.333884:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335646336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019109.333917:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.333921:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019109.333923:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019109.333927:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.333931:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.333934:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019109.333944:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019109.333954:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335646464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019109.333964:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.333967:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019109.333969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019109.333972:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.333975:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.333978:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019109.333983:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019109.333990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335646592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019109.333999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.334004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019109.334006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019109.334010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.334012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.334016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019109.334019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019109.334029:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335646720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019109.334038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.334041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019109.334043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019109.334045:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335646336:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019109.334046:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.334049:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.334052:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019109.334056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019109.334065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335646848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019109.334073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.334076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019109.334077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019109.334081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.334083:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.334086:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019109.334091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019109.334098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335646976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019109.334106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019109.334109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019109.334111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019109.334114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3373 00000100:00080000:2.0:1734019109.334127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019109.334130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019109.334134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019109.334140:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019109.334143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335647104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019109.334170:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86dca80 x1818247335646336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334192:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335646336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334205:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334209:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334215:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335646336:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019109.334223:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335646464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019109.334241:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019109.334247:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86dd880 x1818247335646464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335646464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334270:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335646464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019109.334280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335646592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019109.334295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019109.334301:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86df480 x1818247335646592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335646592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334319:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335646592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019109.334355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335646720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019109.334368:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019109.334375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86dd500 x1818247335646720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335646720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334394:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334397:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334400:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335646720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019109.334405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019109.334418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019109.334424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86dd180 x1818247335646848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335646848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334447:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019109.334454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019109.334466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019109.334472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86dc000 x1818247335646976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334482:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335646976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335646976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019109.334500:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335647104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019109.334513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019109.334519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019109/real 1734019109] req@ffff8800a86ddc00 x1818247335647104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019109.334529:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335647104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019109.334536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019109.334538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019109.334542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335647104:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019110.187998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019110.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019111.189923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019111.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019112.191955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019112.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019113.193950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019113.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019114.195969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019114.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019114.341830:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.341838:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019114.341843:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.341852:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.341861:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.341868:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019114.341903:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335647232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019114.341937:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.341941:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019114.341943:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.341947:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.341950:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.341954:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019114.341963:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019114.341973:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335647360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019114.341983:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.341986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019114.341987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.341991:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.341993:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.341997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019114.342001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019114.342009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335647488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019114.342018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.342021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019114.342022:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.342026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.342028:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.342032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019114.342035:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019114.342045:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335647616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019114.342055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.342058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019114.342059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.342063:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.342065:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.342068:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019114.342072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019114.342072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335647232:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019114.342080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335647744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019114.342088:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.342091:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019114.342093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.342096:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.342098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.342102:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019114.342106:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019114.342113:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335647872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019114.342120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019114.342124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019114.342125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019114.342129:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3378 00000100:00080000:2.0:1734019114.342131:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019114.342134:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019114.342138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019114.342145:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335648000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019114.342167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86de680 x1818247335647232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335647232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335647232:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019114.342250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335647360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019114.342269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342276:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86dc700 x1818247335647360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342287:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335647360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342295:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342298:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335647360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019114.342307:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335647488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019114.342322:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342328:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86df100 x1818247335647488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335647488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335647488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019114.342383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335647616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019114.342396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86ddf80 x1818247335647616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335647616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342424:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342427:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335647616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019114.342433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335647744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019114.342446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86dfb80 x1818247335647744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335647744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342469:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335647744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019114.342482:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335647872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019114.342495:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86df800 x1818247335647872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335647872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342572:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335647872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019114.342578:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019114.342595:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019114.342603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019114/real 1734019114] req@ffff8800a86de300 x1818247335648000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019114.342614:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335648000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019114.342622:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019114.342625:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019114.342628:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648000:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019115.197989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019115.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019116.199990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019116.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019117.201942:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019117.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019118.203990:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019118.213935:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019119.205938:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019119.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019119.349762:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.349771:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019119.349777:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.349790:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.349800:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.349810:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019119.349856:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335648128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019119.349890:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.349894:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019119.349896:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.349901:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.349905:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.349909:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019119.349921:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019119.349932:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335648256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019119.349942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.349946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019119.349947:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.349951:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.349954:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.349957:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019119.349962:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019119.349970:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335648384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019119.349978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.349982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019119.349984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.349987:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.349990:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.349994:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019119.349998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019119.349998:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335648128:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019119.350010:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335648512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019119.350019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.350023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019119.350025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.350028:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.350032:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.350035:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019119.350039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019119.350047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335648640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019119.350055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.350058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019119.350060:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.350063:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.350069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.350073:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019119.350077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019119.350081:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019119.350085:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335648768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019119.350093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019119.350096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019119.350097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019119.350101:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3383 00000100:00080000:2.0:1734019119.350104:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019119.350108:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350109:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86dce00 x1818247335648128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019119.350113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019119.350121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335648896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019119.350129:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335648128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350141:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350144:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350150:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335648128:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019119.350157:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335648256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019119.350175:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350182:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86dea00 x1818247335648256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019119.350194:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335648256/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350202:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350204:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350208:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335648256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019119.350213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335648384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019119.350228:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86dc380 x1818247335648384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019119.350252:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335648384/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335648384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019119.350293:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335648512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019119.350305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86de300 x1818247335648512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019119.350321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335648512/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335648512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019119.350340:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019119.350352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86df800 x1818247335648640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019119.350368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335648640/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019119.350389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019119.350401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350407:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86dfb80 x1818247335648768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019119.350417:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335648768/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019119.350463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019119.350479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019119.350486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019119/real 1734019119] req@ffff8800a86ddf80 x1818247335648896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019119.350496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335648896/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019119.350504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019119.350506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019119.350509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335648896:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019120.207994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019120.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019121.209998:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019121.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019122.211993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019122.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734019123.213941:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019123.214004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019124.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019124.215944:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019124.357840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.357848:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019124.357853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019124.357862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.357871:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.357879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019124.357914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335649024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019124.357948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.357952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019124.357954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019124.357959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.357962:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.357966:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019124.357975:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019124.357986:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335649152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019124.357996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.358001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019124.358003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019124.358006:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.358009:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.358012:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019124.358017:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019124.358024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335649280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019124.358033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.358036:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019124.358037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019124.358041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.358043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.358047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019124.358050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019124.358060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335649408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019124.358070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.358073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019124.358074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019124.358076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335649024:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019124.358078:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.358080:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.358084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019124.358089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019124.358097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335649536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019124.358106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.358108:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019124.358110:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019124.358113:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.358116:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.358119:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019124.358123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019124.358130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335649664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019124.358138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019124.358141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019124.358142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019124.358145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3388 00000100:00080000:2.0:1734019124.358158:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019124.358161:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019124.358165:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019124.358169:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019124.358173:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335649792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019124.358197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86df100 x1818247335649024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335649024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335649024:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019124.358249:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335649152:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019124.358268:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019124.358275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86dc700 x1818247335649152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335649152/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335649152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019124.358306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335649280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019124.358321:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019124.358327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86de680 x1818247335649280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358386:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335649280/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335649280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019124.358432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335649408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019124.358449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019124.358456:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86ddc00 x1818247335649408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335649408/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335649408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019124.358487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335649536:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019124.358499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019124.358505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86dc000 x1818247335649536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335649536/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335649536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019124.358535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335649664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019124.358548:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019124.358554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86dd180 x1818247335649664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335649664/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358574:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358577:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335649664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019124.358583:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335649792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019124.358597:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019124.358603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019124/real 1734019124] req@ffff8800a86dd500 x1818247335649792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019124.358612:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335649792/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019124.358620:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019124.358622:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019124.358625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335649792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019125.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019125.217993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019126.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019126.220000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019127.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019127.221957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019128.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019128.223940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019129.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019129.225962:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019129.365821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.365829:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019129.365834:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019129.365843:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.365853:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.365860:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019129.365896:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335649920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019129.365928:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.365932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019129.365934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019129.365939:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.365942:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.365946:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019129.365955:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019129.365965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335650048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019129.365974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.365978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019129.365979:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019129.365983:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.365985:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.365989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019129.365994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019129.366002:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335650176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019129.366011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.366014:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019129.366016:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019129.366019:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.366021:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.366025:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019129.366029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019129.366038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335650304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019129.366048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.366051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019129.366053:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019129.366054:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335649920:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019129.366056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.366059:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.366062:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019129.366066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019129.366075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335650432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019129.366083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.366086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019129.366088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019129.366092:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.366094:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.366097:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019129.366101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019129.366108:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335650560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019129.366116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019129.366119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019129.366121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019129.366124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3393 00000100:00080000:2.0:1734019129.366136:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019129.366139:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019129.366143:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019129.366145:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019129.366151:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335650688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019129.366175:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86df480 x1818247335649920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366197:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335649920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366210:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366214:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366219:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335649920:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019129.366227:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019129.366246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019129.366300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86dd880 x1818247335650048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335650048/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019129.366336:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019129.366355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019129.366363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86dca80 x1818247335650176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366374:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335650176/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366382:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019129.366418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019129.366432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019129.366438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86dd500 x1818247335650304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335650304/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366457:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019129.366469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335650432:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019129.366481:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019129.366487:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86dd180 x1818247335650432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366497:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335650432/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335650432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019129.366523:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335650560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019129.366535:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019129.366541:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86dc000 x1818247335650560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366551:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335650560/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335650560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019129.366570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335650688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019129.366583:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019129.366589:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019129/real 1734019129] req@ffff8800a86ddc00 x1818247335650688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019129.366599:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335650688/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019129.366607:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019129.366609:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019129.366612:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335650688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019130.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019130.227953:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019131.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019131.229923:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019132.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019132.232014:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019133.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019133.233959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019134.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019134.235972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019134.373835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.373845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019134.373850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.373862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.373874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.373883:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019134.373931:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335650816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019134.373968:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.373973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019134.373975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.373979:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.373983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.373987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019134.373997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019134.374009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335650944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019134.374019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.374022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019134.374024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.374028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.374031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.374035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019134.374040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019134.374049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335651072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019134.374057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.374063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019134.374064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.374068:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.374071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.374075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019134.374079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019134.374080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335650816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019134.374089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335651200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019134.374098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.374101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019134.374103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.374106:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.374109:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.374113:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019134.374117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019134.374126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335651328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019134.374134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.374138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019134.374139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.374143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.374155:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.374159:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019134.374161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734019134.374164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019134.374219:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335651456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019134.374228:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019134.374232:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734019134.374233:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86de680 x1818247335650816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019134.374234:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019134.374238:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3398 00000100:00080000:2.0:1734019134.374241:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019134.374244:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019134.374249:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019134.374253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335650816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734019134.374257:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335651584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019134.374265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374274:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335650816:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019134.374281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650944:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019134.374303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019134.374310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86dc700 x1818247335650944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019134.374322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335650944/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019134.374329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335650944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019134.374345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019134.374360:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019134.374366:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86df100 x1818247335651072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019134.374377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335651072/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019134.374385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374387:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374390:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019134.374418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019134.374431:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019134.374437:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86ddf80 x1818247335651200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019134.374447:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335651200/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019134.374454:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374461:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019134.374466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335651328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019134.374477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019134.374483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86dfb80 x1818247335651328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019134.374494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335651328/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019134.374502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335651328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019134.374512:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335651456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019134.374523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019134.374529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86df800 x1818247335651456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019134.374539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335651456/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019134.374546:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374548:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374551:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335651456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019134.374556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335651584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019134.374568:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019134.374574:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019134/real 1734019134] req@ffff8800a86de300 x1818247335651584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019134.374583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335651584/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019134.374591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019134.374593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019134.374597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335651584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019135.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019135.237979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019136.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019136.239944:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019137.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019137.241925:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019138.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019138.244000:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019139.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019139.245943:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019139.381839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.381849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019139.381853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.381863:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00080000:2.0:1734019139.381873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.381881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019139.381918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335651712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019139.381961:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.381965:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019139.381967:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.381971:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00080000:2.0:1734019139.381974:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.381978:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019139.381988:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019139.381998:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335651840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019139.382007:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.382010:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019139.382011:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.382015:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00080000:2.0:1734019139.382018:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.382021:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019139.382026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019139.382034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335651968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019139.382042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.382045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019139.382047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.382051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00080000:2.0:1734019139.382053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.382057:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019139.382061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019139.382071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335652096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019139.382112:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.382117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019139.382118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.382123:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00100000:3.0:1734019139.382123:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335651712:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019139.382126:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.382130:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019139.382135:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019139.382144:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335652224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019139.382153:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.382157:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019139.382158:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.382161:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00080000:2.0:1734019139.382164:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.382167:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019139.382171:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019139.382178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335652352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019139.382186:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019139.382189:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019139.382191:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019139.382194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3403 00000100:00080000:2.0:1734019139.382207:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019139.382210:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019139.382214:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019139.382214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019139.382222:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335652480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019139.382244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86dc380 x1818247335651712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335651712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335651712:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019139.382298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019139.382316:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019139.382323:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86dea00 x1818247335651840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382334:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335651840/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382343:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382350:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019139.382357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019139.382372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019139.382378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86dce00 x1818247335651968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335651968/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382397:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335651968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019139.382431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019139.382445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019139.382451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86de300 x1818247335652096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335652096/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019139.382481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335652224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019139.382493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019139.382500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86df800 x1818247335652224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335652224/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335652224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019139.382530:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335652352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019139.382542:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019139.382548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86dfb80 x1818247335652352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382557:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335652352/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382565:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382567:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335652352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019139.382576:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335652480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019139.382589:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019139.382595:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019139/real 1734019139] req@ffff8800a86ddf80 x1818247335652480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019139.382606:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335652480/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019139.382621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019139.382624:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019139.382627:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335652480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019140.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019140.248018:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019141.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019141.250013:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019142.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019142.251974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019143.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019143.253951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019144.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019144.255956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019144.389815:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.389824:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019144.389829:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.389840:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.389850:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.389857:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019144.389893:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335652608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019144.389928:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.389932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019144.389934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.389939:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.389943:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.389947:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019144.389956:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019144.389967:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335652736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019144.389977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.389980:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019144.389982:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.389985:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.389988:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.389991:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019144.390023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019144.390034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335652864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019144.390044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.390047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019144.390049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.390052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.390055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.390058:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019144.390062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019144.390073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335652992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019144.390074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335652608:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019144.390089:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.390092:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019144.390094:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.390098:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.390109:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.390113:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019144.390117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019144.390125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335653120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019144.390134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.390137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019144.390139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.390142:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.390152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.390156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019144.390159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019144.390164:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019144.390167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335653248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019144.390175:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019144.390178:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019144.390179:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019144.390183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3408 00000100:00080000:2.0:1734019144.390189:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019144.390192:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390195:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86df100 x1818247335652608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019144.390196:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019144.390204:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335653376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019144.390217:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335652608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390230:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390234:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335652608:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019144.390247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019144.390265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86dc700 x1818247335652736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019144.390284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335652736/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019144.390313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019144.390328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86de680 x1818247335652864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019144.390344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335652864/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019144.390388:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019144.390402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86ddc00 x1818247335652992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019144.390419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335652992/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390429:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390432:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335652992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019144.390439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335653120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019144.390452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86dc000 x1818247335653120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019144.390469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335653120/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390480:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390483:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335653120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019144.390489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335653248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019144.390500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390506:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86dd180 x1818247335653248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019144.390516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335653248/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335653248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019144.390535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335653376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019144.390547:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019144.390553:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019144/real 1734019144] req@ffff8800a86dd500 x1818247335653376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019144.390564:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335653376/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019144.390573:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019144.390575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019144.390578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335653376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019145.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019145.257948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019146.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019146.260035:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019147.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019147.262037:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019148.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019148.264015:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019149.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019149.266013:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019149.397821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.397830:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019149.397834:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019149.397844:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.397853:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.397860:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019149.397897:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335653504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019149.397969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.397973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019149.397976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019149.397981:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.397984:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.397988:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019149.397998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019149.398011:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335653632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019149.398021:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.398024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019149.398026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019149.398029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.398032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.398035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019149.398039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019149.398047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335653760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019149.398056:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.398059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019149.398060:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019149.398064:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.398066:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.398070:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019149.398074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019149.398083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335653888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019149.398093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.398096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019149.398098:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019149.398099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335653504:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019149.398102:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.398104:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.398108:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019149.398112:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019149.398120:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335654016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019149.398128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.398131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019149.398132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019149.398135:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.398138:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.398141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019149.398145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019149.398152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335654144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019149.398160:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019149.398162:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019149.398164:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019149.398167:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3413 00000100:00080000:2.0:1734019149.398180:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019149.398183:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019149.398187:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019149.398194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019149.398196:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335654272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019149.398224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86dca80 x1818247335653504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398246:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335653504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398259:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335653504:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019149.398276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335653632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019149.398294:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019149.398301:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86dd880 x1818247335653632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335653632/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335653632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019149.398342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335653760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019149.398356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019149.398362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86df480 x1818247335653760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335653760/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335653760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019149.398415:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335653888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019149.398428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019149.398435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86dd500 x1818247335653888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335653888/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398454:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335653888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019149.398467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019149.398480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019149.398486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86dd180 x1818247335654016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335654016/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019149.398514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019149.398527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019149.398533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86dc000 x1818247335654144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335654144/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398551:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019149.398562:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019149.398578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019149.398584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019149/real 1734019149] req@ffff8800a86ddc00 x1818247335654272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019149.398594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335654272/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019149.398602:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019149.398604:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019149.398607:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019150.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019150.268010:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019151.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019151.269994:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019152.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019152.272043:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019153.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019153.273964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019154.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019154.275997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019154.405865:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.405873:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019154.405877:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.405887:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.405896:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019154.405903:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019154.405938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335654400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019154.405969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.405974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019154.405976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.405980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.405983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019154.405986:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019154.405995:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019154.406005:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335654528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019154.406015:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.406018:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019154.406020:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.406024:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.406026:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019154.406029:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019154.406034:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019154.406041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335654656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019154.406050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.406053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019154.406055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.406058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.406060:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019154.406064:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019154.406068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019154.406077:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335654784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019154.406086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.406089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019154.406091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.406094:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.406097:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019154.406099:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335654400:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019154.406101:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019154.406104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019154.406112:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335654912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019154.406121:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.406124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019154.406125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.406129:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.406131:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019154.406135:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019154.406139:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019154.406146:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335655040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019154.406153:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019154.406156:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019154.406157:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019154.406161:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3418 00000100:00080000:2.0:1734019154.406178:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019154.406182:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019154.406186:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019154.406191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019154.406194:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335655168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019154.406221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86de680 x1818247335654400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335654400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335654400:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019154.406273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335654528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019154.406296:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019154.406303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86dc700 x1818247335654528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335654528/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335654528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019154.406335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335654656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019154.406350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019154.406356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86df100 x1818247335654656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335654656/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406374:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335654656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019154.406409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335654784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019154.406422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019154.406429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86ddf80 x1818247335654784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335654784/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406455:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335654784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019154.406466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654912:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019154.406478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019154.406484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86dfb80 x1818247335654912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335654912/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406501:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406504:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406507:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335654912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019154.406513:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019154.406525:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019154.406531:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86df800 x1818247335655040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406541:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335655040/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019154.406561:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019154.406574:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019154.406580:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019154/real 1734019154] req@ffff8800a86de300 x1818247335655168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019154.406590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335655168/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019154.406597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019154.406599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019154.406602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019155.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019155.278035:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019156.213875:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019156.280004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019157.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019157.281980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019158.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019158.283993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019159.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019159.285985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019159.413858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.413867:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019159.413872:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.413882:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.413891:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.413898:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019159.413936:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335655296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019159.413970:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.413974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019159.413976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.413980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.413983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.413987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019159.413997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019159.414007:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335655424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019159.414016:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.414020:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019159.414021:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.414025:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.414027:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.414031:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019159.414035:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019159.414043:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335655552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019159.414059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.414062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019159.414064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.414067:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.414070:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.414073:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019159.414077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019159.414087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335655680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019159.414094:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335655296:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019159.414097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.414100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019159.414101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.414105:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.414107:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.414110:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019159.414115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019159.414122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335655808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019159.414130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.414133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019159.414134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.414138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.414140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.414144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019159.414148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019159.414155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335655936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019159.414163:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019159.414166:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019159.414167:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019159.414170:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3423 00000100:00080000:2.0:1734019159.414174:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019159.414177:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019159.414180:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019159.414187:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019159.414188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335656064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019159.414217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86dce00 x1818247335655296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335655296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414257:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414262:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335655296:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019159.414271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335655424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019159.414290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019159.414297:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86dea00 x1818247335655424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335655424/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335655424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019159.414329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335655552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019159.414344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019159.414350:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86dc380 x1818247335655552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335655552/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414374:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335655552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019159.414404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335655680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019159.414421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019159.414428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86de300 x1818247335655680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335655680/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335655680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019159.414459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019159.414471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019159.414477:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86df800 x1818247335655808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414487:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335655808/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019159.414506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019159.414518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019159.414524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86dfb80 x1818247335655936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414541:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335655936/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414549:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335655936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019159.414560:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019159.414573:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019159.414579:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019159/real 1734019159] req@ffff8800a86ddf80 x1818247335656064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019159.414589:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335656064/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019159.414596:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019159.414598:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019159.414601:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019160.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019160.288007:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019161.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019161.290000:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019162.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019162.291965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019163.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019163.294022:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019164.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019164.295979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019164.421832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.421841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019164.421846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.421855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00080000:2.0:1734019164.421864:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.421871:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019164.421906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335656192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019164.421940:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.421944:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019164.421946:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.421951:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00080000:2.0:1734019164.421954:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.421957:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019164.421967:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019164.421977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335656320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019164.421987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.421990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019164.421992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.421995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00080000:2.0:1734019164.421998:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.422001:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019164.422006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019164.422014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335656448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019164.422022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.422025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019164.422027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.422030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00080000:2.0:1734019164.422033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.422036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019164.422040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019164.422049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335656576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019164.422058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.422062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019164.422063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.422067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00100000:3.0:1734019164.422067:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335656192:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019164.422069:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.422073:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019164.422077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019164.422085:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335656704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019164.422092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.422095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019164.422096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.422100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00080000:2.0:1734019164.422102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.422106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019164.422109:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019164.422117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335656832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019164.422125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019164.422128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019164.422129:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019164.422132:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3428 00000100:00080000:2.0:1734019164.422147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019164.422150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019164.422155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019164.422162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335656960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019164.422192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335656320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019164.422217:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335656448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019164.422246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335656576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019164.422265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019164.422282:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019164.422298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019165.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019165.298002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019166.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019166.300009:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019167.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019167.301967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734019167.427953:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019167.427997:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019167.428016:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019167.428051:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019167.428060:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019167.428069:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019167.428076:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019167.428084:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019167.428091:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019167.428178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86ddc00 x1818247335656576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335656576/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335656576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019167.428277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86dc000 x1818247335656704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428299:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335656704/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428313:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428317:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019167.428335:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86dd180 x1818247335656832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335656832/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428368:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019167.428389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86dd500 x1818247335656960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428409:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335656960/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428427:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335656960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019167.428445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86df100 x1818247335656192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335656192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335656192:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019167.428499:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86dc700 x1818247335656320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335656320/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335656320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019167.428556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019164/real 1734019167] req@ffff8800a86de680 x1818247335656448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019167.428575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335656448/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019167.428587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019167.428592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019167.428596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335656448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019168.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019168.303956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019169.213930:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019169.305981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019169.429787:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.429795:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019169.429800:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.429809:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00080000:2.0:1734019169.429818:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.429826:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019169.429861:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335657088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019169.429896:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.429900:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019169.429902:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.429906:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00080000:2.0:1734019169.429909:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.429913:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019169.429922:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019169.429933:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335657216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019169.429942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.429946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019169.429948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.429951:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00080000:2.0:1734019169.429954:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.429958:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019169.429962:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019169.429969:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335657344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019169.429978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.429981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019169.429982:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.429986:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00080000:2.0:1734019169.429988:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.429992:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019169.429996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019169.430006:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335657472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019169.430015:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.430018:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019169.430020:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.430024:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00100000:3.0:1734019169.430024:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335657088:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019169.430026:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.430029:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019169.430034:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019169.430043:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335657600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019169.430051:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.430054:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019169.430055:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.430059:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00080000:2.0:1734019169.430061:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.430065:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019169.430069:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019169.430076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335657728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019169.430084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019169.430086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019169.430088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019169.430091:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3433 00000100:00080000:2.0:1734019169.430103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019169.430106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019169.430110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019169.430116:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019169.430118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335657856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019169.430145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86df480 x1818247335657088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430167:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335657088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430180:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430185:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430190:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335657088:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019169.430198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335657216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019169.430218:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019169.430224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86dd880 x1818247335657216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335657216/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335657216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019169.430257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335657344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019169.430271:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019169.430277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86dca80 x1818247335657344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430287:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335657344/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335657344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019169.430329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335657472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019169.430347:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019169.430353:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86de680 x1818247335657472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335657472/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335657472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019169.430383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335657600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019169.430395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019169.430401:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86dc700 x1818247335657600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430411:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335657600/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430421:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430424:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335657600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019169.430429:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335657728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019169.430441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019169.430447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86df100 x1818247335657728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430464:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335657728/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335657728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019169.430484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335657856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019169.430497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019169.430503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019169/real 1734019169] req@ffff8800a86dd500 x1818247335657856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019169.430513:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335657856/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019169.430520:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019169.430522:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019169.430525:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335657856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019170.213790:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019170.308037:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019171.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019171.309998:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019172.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019172.311983:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019173.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019173.313954:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019174.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019174.315990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019174.437835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.437843:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019174.437847:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019174.437857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.437866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.437873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019174.437910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335657984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019174.437942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.437947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019174.437949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019174.437953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.437956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.437959:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019174.437969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019174.437979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335658112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019174.437989:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.437992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019174.437994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019174.437997:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.438000:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.438003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019174.438008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019174.438016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335658240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019174.438026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.438029:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019174.438030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019174.438033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.438036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.438039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019174.438043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019174.438052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335658368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019174.438061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.438064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019174.438066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019174.438069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335657984:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019174.438070:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.438072:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.438076:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019174.438080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019174.438088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335658496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019174.438097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.438099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019174.438101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019174.438105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.438108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.438111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019174.438115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019174.438122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335658624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019174.438129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019174.438132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019174.438134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019174.438137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3438 00000100:00080000:2.0:1734019174.438149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019174.438152:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019174.438156:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019174.438159:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019174.438164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335658752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019174.438190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86dd180 x1818247335657984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335657984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438226:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335657984:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019174.438243:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335658112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019174.438261:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019174.438268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86dc000 x1818247335658112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438280:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335658112/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335658112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019174.438300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335658240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019174.438314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019174.438320:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86ddc00 x1818247335658240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438330:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335658240/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335658240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019174.438375:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335658368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019174.438389:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019174.438395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86ddf80 x1818247335658368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438406:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335658368/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438416:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335658368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019174.438424:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335658496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019174.438437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019174.438443:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86dfb80 x1818247335658496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438453:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335658496/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438460:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438463:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438466:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335658496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019174.438472:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335658624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019174.438539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019174.438548:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86df800 x1818247335658624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335658624/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438568:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335658624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019174.438580:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335658752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019174.438596:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019174.438602:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019174/real 1734019174] req@ffff8800a86de300 x1818247335658752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019174.438613:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335658752/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019174.438621:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019174.438623:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019174.438626:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335658752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019175.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019175.317993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019176.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019176.319957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019177.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019177.321998:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019178.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019178.323999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019179.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019179.325938:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019179.445796:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.445805:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019179.445809:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.445819:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.445828:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.445836:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019179.445871:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335658880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019179.445904:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.445909:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019179.445911:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.445915:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.445918:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.445922:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019179.445932:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019179.445942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335659008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019179.445952:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.445955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019179.445957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.445960:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.445963:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.445966:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019179.445970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019179.445978:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335659136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019179.445987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.445992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019179.445994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.445997:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.446000:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.446003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019179.446007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019179.446017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335659264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019179.446026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.446029:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019179.446029:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335658880:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019179.446031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.446035:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.446037:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.446041:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019179.446045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019179.446053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335659392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019179.446061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.446064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019179.446066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.446069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.446072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.446075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019179.446080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019179.446087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335659520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019179.446095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019179.446098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019179.446099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019179.446102:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3443 00000100:00080000:2.0:1734019179.446114:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019179.446118:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019179.446122:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019179.446123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019179.446130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335659648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019179.446154:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86dc380 x1818247335658880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446177:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335658880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446190:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446194:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446199:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335658880:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019179.446207:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019179.446225:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019179.446232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86dea00 x1818247335659008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335659008/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019179.446264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019179.446279:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019179.446285:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86dce00 x1818247335659136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446295:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335659136/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019179.446339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019179.446352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019179.446358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86de300 x1818247335659264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335659264/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019179.446389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335659392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019179.446449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019179.446458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86df800 x1818247335659392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335659392/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335659392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019179.446490:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335659520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019179.446507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019179.446513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86dfb80 x1818247335659520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446524:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335659520/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335659520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019179.446543:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335659648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019179.446556:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019179.446562:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019179/real 1734019179] req@ffff8800a86ddf80 x1818247335659648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019179.446572:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335659648/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019179.446579:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019179.446582:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019179.446585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335659648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019180.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019180.327980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019181.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019181.329987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019182.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019182.331988:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019183.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019183.333945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019184.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019184.335955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019184.453839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.453849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019184.453855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.453867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.453878:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019184.453887:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019184.453937:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335659776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019184.453981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.453985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019184.453987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.453992:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.453995:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019184.453999:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019184.454011:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019184.454022:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335659904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019184.454031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.454034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019184.454036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.454040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.454043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019184.454047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019184.454052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019184.454060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335660032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019184.454068:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.454071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019184.454073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.454076:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.454079:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019184.454084:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019184.454088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019184.454090:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335659776:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019184.454099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335660160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019184.454108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.454111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019184.454113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.454116:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.454119:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019184.454122:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019184.454127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019184.454135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335660288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019184.454143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.454146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019184.454148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.454151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.454163:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019184.454166:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019184.454171:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019184.454172:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019184.454179:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335660416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019184.454187:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019184.454190:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019184.454192:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019184.454196:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3448 00000100:00080000:2.0:1734019184.454199:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019184.454200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86ddc00 x1818247335659776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019184.454202:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019184.454206:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019184.454214:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335660544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019184.454220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335659776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335659776:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019184.454247:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019184.454265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019184.454272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86dc000 x1818247335659904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019184.454284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335659904/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454291:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454297:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335659904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019184.454303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019184.454348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019184.454356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86dd180 x1818247335660032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019184.454369:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335660032/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019184.454410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019184.454426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019184.454432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86dd500 x1818247335660160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019184.454442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335660160/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019184.454461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335660288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019184.454473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019184.454479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86df100 x1818247335660288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019184.454489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335660288/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454500:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454503:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335660288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019184.454509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335660416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019184.454521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019184.454526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86dc700 x1818247335660416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019184.454536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335660416/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335660416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019184.454554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335660544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019184.454567:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019184.454573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019184/real 1734019184] req@ffff8800a86de680 x1818247335660544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019184.454583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335660544/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019184.454591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019184.454593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019184.454597:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335660544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019185.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019185.337993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019186.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019186.340003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019187.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019187.341981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019188.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019188.343924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019189.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019189.345976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019189.461926:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.461934:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019189.461939:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019189.461949:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3453 00000100:00080000:0.0:1734019189.461958:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.461965:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019189.462000:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335660672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019189.462038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.462043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019189.462044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019189.462049:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3453 00000100:00080000:0.0:1734019189.462052:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.462055:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019189.462066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019189.462076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335660800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019189.462086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.462089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019189.462091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019189.462094:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3453 00000100:00080000:0.0:1734019189.462096:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.462100:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019189.462104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019189.462114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335660928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019189.462123:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.462128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019189.462130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019189.462134:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3453 00000100:00080000:0.0:1734019189.462137:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.462140:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019189.462144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019189.462152:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335661056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019189.462162:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.462165:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019189.462166:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019189.462167:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335660672:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019189.462170:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3453 00000100:00080000:0.0:1734019189.462172:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.462176:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019189.462180:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019189.462188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335661184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019189.462196:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.462199:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019189.462200:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019189.462204:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3453 00000100:00080000:0.0:1734019189.462206:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.462210:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019189.462214:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019189.462222:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335661312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019189.462278:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019189.462283:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019189.462285:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019189.462289:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3453 00000800:00000100:3.0:1734019189.462291:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019189.462294:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019189.462298:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019189.462303:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019189.462314:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335661440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019189.462322:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59c8e00 x1818247335660672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335660672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335660672:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019189.462374:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019189.462395:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019189.462402:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59c8a80 x1818247335660800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462413:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335660800/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019189.462434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019189.462449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019189.462457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59cb480 x1818247335660928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335660928/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335660928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019189.462514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019189.462527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019189.462534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59ca300 x1818247335661056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335661056/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462558:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019189.462564:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335661184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019189.462577:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019189.462583:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59caa00 x1818247335661184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462593:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335661184/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462609:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462611:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462614:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335661184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019189.462620:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335661312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019189.462632:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019189.462638:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59cb100 x1818247335661312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462648:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335661312/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462656:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462658:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462661:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335661312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019189.462666:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335661440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019189.462679:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019189.462686:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019189/real 1734019189] req@ffff8800b59c8000 x1818247335661440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019189.462891:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335661440/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019189.462900:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019189.462902:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019189.462906:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335661440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019190.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019190.347979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019191.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019191.349999:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019192.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019192.351995:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019193.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019193.353938:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019194.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019194.355969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019194.469844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.469852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019194.469857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019194.469866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.469876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.469883:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019194.469919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335661568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019194.469953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.469957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019194.469959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019194.469963:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.469967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.469971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019194.469980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019194.469991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335661696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019194.470001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.470004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019194.470006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019194.470009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.470012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.470015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019194.470019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019194.470027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335661824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019194.470035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.470038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019194.470040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019194.470043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.470046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.470049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019194.470053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019194.470060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335661952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019194.470076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.470079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019194.470080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019194.470081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335661568:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019194.470084:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.470087:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.470090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019194.470094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019194.470102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335662080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019194.470110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.470112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019194.470114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019194.470117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.470119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.470123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019194.470126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019194.470133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335662208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019194.470141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019194.470144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019194.470181:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019194.470186:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3459 00000100:00080000:0.0:1734019194.470189:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019194.470193:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019194.470197:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019194.470208:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335662336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019194.470355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59c9c00 x1818247335661568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470409:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335661568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335661568:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019194.470440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019194.470461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59cad80 x1818247335661696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335661696/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470489:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019194.470506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019194.470521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59c8700 x1818247335661824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335661824/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019194.470587:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019194.470601:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470607:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59cb800 x1818247335661952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470617:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335661952/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470625:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470627:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470631:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335661952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019194.470637:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019194.470649:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470655:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59c9500 x1818247335662080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470667:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335662080/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470675:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470677:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470680:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019194.470685:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019194.470803:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470811:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59ca680 x1818247335662208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470822:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335662208/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470830:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470832:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470835:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019194.470841:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019194.470856:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019194.470862:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019194/real 1734019194] req@ffff8800b59cbb80 x1818247335662336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019194.470873:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335662336/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019194.470881:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019194.470884:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019194.470888:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019195.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019195.357978:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019196.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019196.360000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019197.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019197.362012:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019198.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019198.363998:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019199.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019199.366007:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019199.477829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.477838:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019199.477842:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019199.477852:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.477860:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.477868:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019199.477902:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335662464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019199.477936:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.477940:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019199.477942:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019199.477946:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.477949:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.477953:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019199.477962:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019199.477973:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335662592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019199.477982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.477985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019199.477988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019199.477991:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.477994:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.477997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019199.478002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019199.478011:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335662720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019199.478020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.478023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019199.478024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019199.478028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.478030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.478034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019199.478038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019199.478045:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335662848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019199.478055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.478058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019199.478096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019199.478098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335662464:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019199.478102:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.478105:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.478109:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019199.478114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019199.478123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335662976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019199.478132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.478135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019199.478137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019199.478140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.478143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.478146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019199.478150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019199.478157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335663104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019199.478165:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019199.478167:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019199.478169:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019199.478172:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3464 00000100:00080000:0.0:1734019199.478175:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019199.478178:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019199.478182:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019199.478190:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335663232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019199.478193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59c9180 x1818247335662464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335662464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335662464:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019199.478277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335662592:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019199.478296:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59c9f80 x1818247335662592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335662592/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478327:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478332:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335662592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019199.478339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335662720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019199.478355:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478361:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59cbb80 x1818247335662720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478371:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335662720/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478380:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478385:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335662720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019199.478420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335662848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019199.478433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59ca680 x1818247335662848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478449:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335662848/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335662848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019199.478469:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019199.478482:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478489:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59c9500 x1818247335662976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335662976/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335662976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019199.478518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335663104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019199.478531:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478537:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59cb800 x1818247335663104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478546:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335663104/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478554:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335663104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019199.478565:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335663232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019199.478578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019199.478584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019199/real 1734019199] req@ffff8800b59c8700 x1818247335663232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019199.478595:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335663232/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019199.478610:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019199.478613:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019199.478616:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335663232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019200.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019200.367955:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019201.213817:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019201.370009:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019202.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019202.372001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019203.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019203.373969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019204.213790:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019204.376006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019204.485801:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.485810:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019204.485815:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.485825:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00080000:0.0:1734019204.485834:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.485843:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019204.485881:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335663360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019204.485915:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.485919:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019204.485921:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.485926:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00080000:0.0:1734019204.485929:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.485932:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019204.485941:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019204.485952:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335663488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019204.485962:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.485965:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019204.485966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.485970:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00080000:0.0:1734019204.485973:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.486002:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019204.486007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019204.486016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335663616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019204.486025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.486028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019204.486031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.486035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00080000:0.0:1734019204.486038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.486041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019204.486045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019204.486053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335663744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019204.486062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.486065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019204.486067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.486071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00100000:3.0:1734019204.486072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335663360:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019204.486073:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.486077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019204.486081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019204.486089:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335663872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019204.486098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.486101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019204.486102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.486106:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00080000:0.0:1734019204.486108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.486111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019204.486115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019204.486122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335664000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019204.486131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019204.486134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019204.486136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019204.486139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3469 00000100:00080000:0.0:1734019204.486142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019204.486145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019204.486149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019204.486157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335664128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019204.486167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59cad80 x1818247335663360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335663360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335663360:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019204.486250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335663488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019204.486269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59c9c00 x1818247335663488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335663488/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335663488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019204.486319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335663616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019204.486334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486340:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59c8000 x1818247335663616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335663616/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335663616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019204.486398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335663744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019204.486411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59cb100 x1818247335663744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335663744/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486439:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486442:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335663744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019204.486449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335663872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019204.486462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59caa00 x1818247335663872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335663872/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335663872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019204.486497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019204.486509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59ca300 x1818247335664000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335664000/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486532:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486537:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019204.486543:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019204.486555:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019204.486561:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019204/real 1734019204] req@ffff8800b59cb480 x1818247335664128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019204.486573:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335664128/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019204.486581:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019204.486583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019204.486586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019205.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019205.377929:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019206.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019206.380022:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019207.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019207.382013:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019208.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019208.384023:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019209.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019209.385979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019209.493820:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019209.493829:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019209.493834:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.493844:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.493854:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.493862:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019209.493939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335664256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019209.493973:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019209.493977:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019209.493979:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.493984:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.493987:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.493991:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019209.494001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019209.494013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335664384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019209.494023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019209.494027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019209.494028:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.494032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.494034:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.494038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019209.494042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019209.494052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335664512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019209.494061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019209.494064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019209.494066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.494069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.494071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.494075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019209.494079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019209.494086:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335664640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019209.494097:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019209.494098:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335664256:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019209.494100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019209.494101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.494105:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.494107:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.494111:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019209.494115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019209.494123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335664768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019209.494130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019209.494133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019209.494135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.494138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.494140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.494144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019209.494148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019209.494155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335664896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019209.494163:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019209.494165:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019209.494167:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019209.494171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3474 00000100:00080000:0.0:1734019209.494174:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019209.494178:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019209.494182:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019209.494188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019209.494190:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335665024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019209.494219:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59c8a80 x1818247335664256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335664256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335664256:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019209.494272:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335664384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019209.494290:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019209.494298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59c8e00 x1818247335664384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494318:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335664384/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335664384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019209.494339:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335664512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019209.494353:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019209.494359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59cb480 x1818247335664512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335664512/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494377:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335664512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019209.494416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335664640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019209.494429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019209.494436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59ca300 x1818247335664640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335664640/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494454:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494458:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494461:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335664640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019209.494467:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664768:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019209.494480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019209.494486:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59caa00 x1818247335664768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494496:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335664768/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494504:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494510:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019209.494516:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019209.494528:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019209.494533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59cb100 x1818247335664896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335664896/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494551:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335664896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019209.494562:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019209.494578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019209.494584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019209/real 1734019209] req@ffff8800b59c8000 x1818247335665024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019209.494594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335665024/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019209.494601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019209.494603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019209.494606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019210.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019210.387970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019211.213950:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019211.390017:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019212.213893:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019212.391991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019213.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019213.393983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019214.213882:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019214.395986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019214.501861:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.501871:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019214.501876:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.501889:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.501899:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.501909:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019214.501956:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335665152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019214.501993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.501997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019214.501999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.502004:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.502007:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.502011:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019214.502022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019214.502034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335665280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019214.502043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.502048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019214.502050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.502054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.502057:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.502060:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019214.502065:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019214.502073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335665408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019214.502082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.502085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019214.502087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.502090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.502093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.502097:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019214.502101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019214.502101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335665152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019214.502109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335665536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019214.502118:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.502121:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019214.502123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.502126:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.502129:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.502132:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019214.502137:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019214.502145:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335665664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019214.502154:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.502157:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019214.502158:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.502162:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.502165:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.502168:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019214.502173:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019214.502180:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335665792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019214.502184:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019214.502188:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019214.502191:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019214.502193:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019214.502197:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3479 00000100:00080000:0.0:1734019214.502199:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019214.502203:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019214.502207:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019214.502212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59c9c00 x1818247335665152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019214.502217:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335665920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019214.502232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335665152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502253:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335665152:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019214.502260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335665280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019214.502281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019214.502288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59cad80 x1818247335665280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019214.502300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335665280/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335665280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019214.502319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335665408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019214.502333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019214.502339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59c8700 x1818247335665408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019214.502349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335665408/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335665408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019214.502392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335665536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019214.502405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019214.502412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59cb800 x1818247335665536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019214.502430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335665536/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335665536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019214.502448:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019214.502459:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019214.502465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59c9500 x1818247335665664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019214.502475:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335665664/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502484:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502487:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019214.502492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019214.502504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019214.502510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59ca680 x1818247335665792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019214.502519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335665792/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019214.502539:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019214.502551:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019214.502557:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019214/real 1734019214] req@ffff8800b59cbb80 x1818247335665920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019214.502566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335665920/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019214.502573:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019214.502575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019214.502578:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335665920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019215.213886:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019215.397953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019216.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019216.400011:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019217.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019217.401991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019218.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019218.404010:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019219.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019219.405992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019219.509839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019219.509849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019219.509856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019219.509870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.509880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019219.509890:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019219.509940:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335666048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019219.509977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019219.509981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019219.509983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019219.509988:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.509992:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019219.509995:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019219.510007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019219.510019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335666176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019219.510028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019219.510032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019219.510034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019219.510038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.510041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019219.510044:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019219.510049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019219.510060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335666304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019219.510077:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019219.510078:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335666048:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019219.510080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019219.510082:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019219.510086:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.510088:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019219.510092:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019219.510096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019219.510105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335666432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019219.510113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019219.510117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019219.510118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019219.510122:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.510124:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019219.510128:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019219.510132:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019219.510140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335666560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019219.510147:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019219.510151:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019219.510152:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019219.510156:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.510160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734019219.510161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019219.510164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019219.510168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019219.510176:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335666688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019219.510184:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019219.510187:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019219.510188:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019219.510189:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59c9f80 x1818247335666048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019219.510192:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3484 00000100:00080000:0.0:1734019219.510195:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019219.510198:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019219.510202:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019219.510209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335666048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019219.510211:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335666816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019219.510221:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510226:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335666048:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019219.510239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335666176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019219.510257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019219.510263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59c9180 x1818247335666176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019219.510274:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335666176/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019219.510283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335666176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019219.510319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335666304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019219.510333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019219.510339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59cbb80 x1818247335666304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019219.510351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335666304/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019219.510359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335666304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019219.510369:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335666432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019219.510385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019219.510391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59ca680 x1818247335666432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019219.510401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335666432/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019219.510408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510410:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335666432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019219.510418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335666560:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019219.510430:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019219.510436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59c9500 x1818247335666560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019219.510446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335666560/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019219.510453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510456:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510459:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335666560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019219.510464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335666688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019219.510476:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019219.510482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59cb800 x1818247335666688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019219.510499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335666688/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019219.510507:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510512:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335666688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019219.510517:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335666816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019219.510529:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019219.510535:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019219/real 1734019219] req@ffff8800b59c8700 x1818247335666816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019219.510545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335666816/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019219.510552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019219.510553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019219.510556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335666816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019220.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019220.407973:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019221.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019221.410017:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019222.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019222.411977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019223.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019223.413960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019224.213907:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019224.415983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019224.517811:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.517820:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019224.517824:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019224.517834:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.517843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.517851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019224.517888:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335666944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019224.517922:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.517927:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019224.517929:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019224.517933:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.517937:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.517940:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019224.517950:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019224.517961:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335667072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019224.517970:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.517974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019224.517975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019224.517979:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.517981:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.517985:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019224.517989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019224.517997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335667200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019224.518005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.518008:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019224.518010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019224.518013:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.518016:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.518019:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019224.518024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019224.518031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335667328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019224.518041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.518045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019224.518046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019224.518046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335666944:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019224.518050:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.518052:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.518056:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019224.518059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019224.518067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335667456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019224.518075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.518078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019224.518079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019224.518083:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.518085:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.518089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019224.518092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019224.518100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335667584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019224.518107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019224.518110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019224.518112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019224.518116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3489 00000100:00080000:0.0:1734019224.518119:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019224.518122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019224.518126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019224.518135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335667712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019224.518137:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518167:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59cad80 x1818247335666944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335666944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518203:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518207:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335666944:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019224.518220:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019224.518238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59c9c00 x1818247335667072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518257:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335667072/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019224.518277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019224.518292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59c8000 x1818247335667200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518308:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335667200/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019224.518358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019224.518372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59cb100 x1818247335667328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335667328/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518399:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518402:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019224.518408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335667456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019224.518420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59caa00 x1818247335667456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335667456/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335667456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019224.518454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335667584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019224.518470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59ca300 x1818247335667584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335667584/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518493:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335667584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019224.518504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335667712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019224.518516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019224.518522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019224/real 1734019224] req@ffff8800b59cb480 x1818247335667712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019224.518531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335667712/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019224.518538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019224.518541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019224.518543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335667712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019225.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019225.417981:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019226.213911:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019226.419985:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019227.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019227.421935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019228.213771:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019228.423967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019229.213948:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019229.425968:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019229.525847:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.525856:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019229.525861:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019229.525871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.525881:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.525889:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019229.525923:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335667840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019229.525958:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.525962:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019229.525964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019229.525969:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.525972:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.525976:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019229.525986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019229.525997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335667968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019229.526008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.526011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019229.526012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019229.526016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.526018:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.526021:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019229.526025:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019229.526036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335668096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019229.526044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.526047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019229.526049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019229.526052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.526055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.526058:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019229.526062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019229.526070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335668224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019229.526080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.526083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019229.526084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019229.526087:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335667840:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019229.526088:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.526091:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.526095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019229.526099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019229.526107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335668352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019229.526115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.526118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019229.526119:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019229.526122:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.526125:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.526128:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019229.526132:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019229.526139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335668480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019229.526147:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019229.526149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019229.526151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019229.526154:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3494 00000100:00080000:0.0:1734019229.526157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019229.526160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019229.526164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019229.526171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335668608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019229.526213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019229.526237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019229.526266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019229.526284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335668352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019229.526302:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335668480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019229.526319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335668608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019230.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019230.427896:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019231.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019231.429963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019232.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019232.431982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734019232.531994:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019232.532041:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019232.532062:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019232.532099:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019232.532107:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019232.532116:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019232.532122:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019232.532129:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019232.532138:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019232.532213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59ca300 x1818247335668224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335668224/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532296:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019232.532310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59caa00 x1818247335668352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335668352/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335668352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019232.532380:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59cb100 x1818247335668480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335668480/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532414:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335668480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019232.532435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59c8000 x1818247335668608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335668608/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335668608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019232.532488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59c8e00 x1818247335667840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335667840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532608:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532613:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532618:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335667840:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019232.532630:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59c8a80 x1818247335667968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532649:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335667968/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532662:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532666:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532672:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335667968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019232.532684:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019229/real 1734019232] req@ffff8800b59cb480 x1818247335668096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019232.532821:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335668096/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019232.532835:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019232.532841:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019232.532857:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019233.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019233.433987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019234.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019234.436020:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019234.533811:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.533820:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019234.533825:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.533835:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00080000:0.0:1734019234.533843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.533851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019234.533887:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335668736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019234.533922:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.533927:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019234.533928:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.533933:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00080000:0.0:1734019234.533936:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.533940:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019234.533949:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019234.533960:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335668864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019234.533969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.533972:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019234.533974:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.533978:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00080000:0.0:1734019234.533981:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.533984:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019234.533989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019234.533997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335668992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019234.534006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.534009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019234.534010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.534014:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00080000:0.0:1734019234.534016:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.534019:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019234.534023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019234.534031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335669120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019234.534040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.534043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019234.534044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.534048:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00100000:3.0:1734019234.534048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335668736:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019234.534050:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.534054:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019234.534058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019234.534066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335669248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019234.534075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.534078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019234.534080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.534084:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00080000:0.0:1734019234.534086:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.534089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019234.534093:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019234.534101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335669376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019234.534108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019234.534111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019234.534112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019234.534116:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3499 00000100:00080000:0.0:1734019234.534118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019234.534121:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019234.534125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019234.534134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335669504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019234.534143:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534173:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59c9c00 x1818247335668736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335668736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534208:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534212:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534217:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335668736:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019234.534226:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019234.534245:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534251:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59cad80 x1818247335668864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534263:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335668864/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534271:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534274:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019234.534284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019234.534299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59c8700 x1818247335668992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335668992/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534333:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534336:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335668992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019234.534372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335669120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019234.534386:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59cb800 x1818247335669120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335669120/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335669120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019234.534423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335669248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019234.534436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59c9500 x1818247335669248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335669248/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335669248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019234.534527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335669376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019234.534544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59ca680 x1818247335669376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335669376/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534572:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534574:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534577:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335669376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019234.534582:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335669504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019234.534595:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019234.534601:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019234/real 1734019234] req@ffff8800b59cbb80 x1818247335669504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019234.534611:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335669504/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019234.534619:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019234.534622:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019234.534625:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335669504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019235.214036:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019235.437959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019236.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019236.439998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019237.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019237.441959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019238.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019238.443989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019239.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019239.445950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019239.541844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.541853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019239.541857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019239.541867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.541879:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.541887:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019239.541924:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335669632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019239.541958:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.541962:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019239.541964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019239.541969:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.541972:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.541975:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019239.541985:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019239.541996:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335669760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019239.542006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.542009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019239.542011:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019239.542014:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.542017:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.542020:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019239.542024:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019239.542034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335669888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019239.542043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.542046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019239.542048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019239.542052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.542055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.542058:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019239.542063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019239.542070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335670016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019239.542079:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.542083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019239.542084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019239.542085:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335669632:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019239.542088:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.542091:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.542095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019239.542098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019239.542106:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335670144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019239.542114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.542117:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019239.542118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019239.542122:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.542124:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.542127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019239.542132:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019239.542139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335670272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019239.542147:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019239.542150:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019239.542152:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019239.542155:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3504 00000100:00080000:0.0:1734019239.542159:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019239.542162:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019239.542165:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019239.542173:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335670400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019239.542178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59c9180 x1818247335669632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335669632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542253:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335669632:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019239.542261:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335669760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019239.542280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59c9f80 x1818247335669760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542298:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335669760/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335669760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019239.542319:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335669888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019239.542333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59cbb80 x1818247335669888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335669888/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542360:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335669888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019239.542442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019239.542461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59ca680 x1818247335670016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335670016/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019239.542498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335670144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019239.542511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59c9500 x1818247335670144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335670144/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335670144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019239.542549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335670272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019239.542561:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542567:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59cb800 x1818247335670272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542577:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335670272/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542587:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542590:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335670272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019239.542595:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335670400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019239.542609:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019239.542615:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019239/real 1734019239] req@ffff8800b59c8700 x1818247335670400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019239.542626:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335670400/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019239.542633:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019239.542635:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019239.542638:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335670400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019240.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019240.447970:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019241.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019241.449997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019242.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019242.451987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019243.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019243.453976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019244.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019244.455940:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019244.549816:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.549826:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019244.549831:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.549844:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.549854:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019244.549863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019244.549907:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335670528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019244.549943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.549948:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019244.549950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.549955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.549958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019244.549961:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019244.549973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019244.549985:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335670656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019244.549996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.549999:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019244.550001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.550005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.550007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019244.550011:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019244.550015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019244.550023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335670784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019244.550032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.550035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019244.550037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.550041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.550043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019244.550045:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335670528:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019244.550047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019244.550051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019244.550058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335670912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019244.550066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.550070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019244.550072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.550076:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.550078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019244.550081:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019244.550085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019244.550093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335671040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019244.550101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.550104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019244.550105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.550109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.550111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019244.550115:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019244.550119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019244.550123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019244.550127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335671168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019244.550134:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019244.550137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019244.550139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019244.550142:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3509 00000100:00080000:0.0:1734019244.550145:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019244.550149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550149:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59cad80 x1818247335670528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734019244.550153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019244.550162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335671296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019244.550170:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335670528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550181:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550185:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550190:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335670528:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019244.550197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019244.550215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59c9c00 x1818247335670656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019244.550233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335670656/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019244.550274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019244.550288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550328:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59cb480 x1818247335670784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019244.550342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335670784/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019244.550361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019244.550376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59c8a80 x1818247335670912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019244.550393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335670912/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335670912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019244.550410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019244.550422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59c8e00 x1818247335671040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019244.550438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335671040/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550458:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019244.550463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019244.550475:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550480:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59c8000 x1818247335671168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019244.550490:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335671168/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550497:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019244.550508:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019244.550520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019244.550526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019244/real 1734019244] req@ffff8800b59cb100 x1818247335671296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019244.550536:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335671296/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019244.550543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019244.550545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019244.550548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019245.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019245.457958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019246.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019246.459946:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019247.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019247.461892:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019248.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019248.463939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019249.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019249.465971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019249.557806:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.557816:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019249.557821:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.557833:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.557843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.557853:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019249.557896:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335671424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019249.557932:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.557937:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019249.557938:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.557943:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.557946:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.557950:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019249.557961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019249.557972:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335671552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019249.557982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.557985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019249.557987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.557991:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.557994:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.557997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019249.558003:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019249.558014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335671680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019249.558023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.558026:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019249.558027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.558031:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.558033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.558037:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019249.558041:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019249.558042:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335671424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019249.558049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335671808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019249.558058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.558061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019249.558063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.558067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.558069:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.558072:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019249.558077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019249.558084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335671936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019249.558093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.558097:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019249.558099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.558103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.558105:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.558109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019249.558113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019249.558120:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019249.558121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335672064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019249.558129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019249.558132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019249.558133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019249.558137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3514 00000100:00080000:0.0:1734019249.558139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019249.558143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558145:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59caa00 x1818247335671424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734019249.558147:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019249.558155:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335672192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019249.558165:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335671424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558176:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558180:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558185:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335671424:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019249.558192:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335671552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019249.558240:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59ca300 x1818247335671552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019249.558262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335671552/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558273:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335671552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019249.558287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335671680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019249.558304:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558311:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59cb100 x1818247335671680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019249.558321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335671680/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335671680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019249.558364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335671808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019249.558377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59c8000 x1818247335671808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019249.558393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335671808/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558400:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335671808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019249.558412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019249.558424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558429:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59c8e00 x1818247335671936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019249.558441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335671936/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335671936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019249.558458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019249.558469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59c8a80 x1818247335672064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019249.558485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335672064/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019249.558502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019249.558513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019249.558519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019249/real 1734019249] req@ffff8800b59cb480 x1818247335672192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019249.558529:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335672192/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019249.558536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019249.558539:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019249.558542:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019250.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019250.467879:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019251.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019251.470018:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019252.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019252.472014:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019253.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019253.473966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019254.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019254.475987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019254.565806:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.565815:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019254.565819:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.565829:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.565838:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.565846:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019254.565883:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335672320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019254.565924:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.565928:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019254.565930:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.565935:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.565938:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.565941:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019254.565951:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019254.565962:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335672448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019254.565971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.565974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019254.565976:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.565979:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.565982:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.565985:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019254.565989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019254.565997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335672576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019254.566006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.566009:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019254.566010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.566014:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.566017:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.566021:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019254.566025:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019254.566033:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335672704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019254.566042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.566045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019254.566046:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335672320:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019254.566047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.566051:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.566053:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.566057:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019254.566061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019254.566068:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335672832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019254.566076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.566079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019254.566080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.566084:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.566086:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.566089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019254.566093:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019254.566101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335672960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019254.566109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019254.566112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019254.566114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019254.566117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3519 00000100:00080000:0.0:1734019254.566121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019254.566164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019254.566169:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019254.566180:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335673088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019254.566181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59c9c00 x1818247335672320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335672320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335672320:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019254.566265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335672448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019254.566286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59cad80 x1818247335672448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335672448/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566313:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335672448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019254.566327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335672576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019254.566342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59c8700 x1818247335672576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335672576/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335672576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019254.566405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335672704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019254.566419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59cb800 x1818247335672704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335672704/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566449:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335672704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019254.566455:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019254.566468:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59c9500 x1818247335672832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335672832/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019254.566505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019254.566517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59ca680 x1818247335672960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335672960/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335672960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019254.566551:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019254.566563:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019254.566569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019254/real 1734019254] req@ffff8800b59cbb80 x1818247335673088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019254.566579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335673088/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019254.566594:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019254.566596:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019254.566600:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019255.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019255.477968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019256.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019256.479996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019257.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019257.481969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019258.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019258.483986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019259.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019259.485933:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019259.573837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.573847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019259.573853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019259.573866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.573877:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.573887:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019259.573935:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335673216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019259.573971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.573975:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019259.573977:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019259.573983:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.573986:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.573989:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019259.574000:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019259.574013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335673344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019259.574024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.574028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019259.574030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019259.574034:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.574036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.574078:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019259.574083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019259.574096:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335673472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019259.574103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335673216:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019259.574105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.574109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019259.574111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019259.574115:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.574118:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.574122:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019259.574126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019259.574136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335673600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019259.574144:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.574148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019259.574150:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019259.574154:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.574158:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.574163:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019259.574167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019259.574175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335673728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019259.574183:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.574186:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019259.574187:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734019259.574188:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019259.574191:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.574194:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.574197:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019259.574201:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019259.574209:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335673856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019259.574215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59c9f80 x1818247335673216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019259.574217:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019259.574220:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019259.574222:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019259.574225:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3524 00000100:00080000:0.0:1734019259.574228:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019259.574231:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019259.574236:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019259.574237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335673216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019259.574245:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335673984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019259.574249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574253:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335673216:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019259.574265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335673344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019259.574282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019259.574289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59c9180 x1818247335673344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019259.574301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335673344/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019259.574317:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335673344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019259.574352:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335673472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019259.574366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019259.574372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59cbb80 x1818247335673472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019259.574383:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335673472/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019259.574391:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574393:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335673472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019259.574401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335673600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019259.574413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019259.574419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59ca680 x1818247335673600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019259.574429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335673600/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019259.574436:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574438:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335673600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019259.574447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019259.574459:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019259.574465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59c9500 x1818247335673728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019259.574475:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335673728/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019259.574483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019259.574493:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019259.574505:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019259.574511:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59cb800 x1818247335673856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019259.574521:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335673856/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019259.574528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574530:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574533:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019259.574538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019259.574550:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019259.574556:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019259/real 1734019259] req@ffff8800b59c8700 x1818247335673984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019259.574567:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335673984/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019259.574575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019259.574577:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019259.574580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335673984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019260.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019260.487949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019261.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019261.489971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019262.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019262.491946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019263.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019263.493962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019264.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019264.496006:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019264.581809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019264.581821:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019264.581825:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.581836:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.581846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.581854:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019264.581891:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335674112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019264.581927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019264.581931:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019264.581933:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.581937:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.581940:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.581944:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019264.581993:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019264.582007:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335674240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019264.582017:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019264.582020:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019264.582022:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.582026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.582029:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.582032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019264.582038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019264.582047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335674368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019264.582056:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019264.582059:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019264.582060:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.582064:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.582066:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.582070:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019264.582074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019264.582081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335674496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019264.582091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019264.582091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335674112:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019264.582094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019264.582096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.582100:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.582102:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.582106:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019264.582110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019264.582117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335674624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019264.582125:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019264.582128:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019264.582130:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.582133:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.582136:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.582139:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019264.582144:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019264.582151:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335674752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019264.582158:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019264.582161:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019264.582163:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019264.582166:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3529 00000100:00080000:0.0:1734019264.582169:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019264.582172:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019264.582176:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019264.582185:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335674880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019264.582185:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59cad80 x1818247335674112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335674112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335674112:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019264.582267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335674240:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019264.582285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59c9c00 x1818247335674240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335674240/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582321:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582327:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335674240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019264.582333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335674368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019264.582348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59cb480 x1818247335674368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335674368/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582374:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335674368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019264.582408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335674496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019264.582421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59c8a80 x1818247335674496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582439:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335674496/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335674496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019264.582460:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335674624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019264.582473:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582479:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59c8e00 x1818247335674624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582489:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335674624/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335674624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019264.582509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335674752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019264.582521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582527:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59c8000 x1818247335674752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335674752/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335674752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019264.582556:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335674880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019264.582573:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019264.582580:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019264/real 1734019264] req@ffff8800b59cb100 x1818247335674880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019264.582590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335674880/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019264.582597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019264.582599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019264.582602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335674880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019265.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019265.497967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019266.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019266.500008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019267.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019267.502014:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019268.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019268.504000:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019269.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019269.505971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019269.589841:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019269.589852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019269.589857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.589896:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.589908:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.589918:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019269.589967:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335675008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019269.590011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019269.590016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019269.590018:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.590023:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.590026:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.590030:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019269.590041:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019269.590053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335675136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019269.590063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019269.590067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019269.590068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.590072:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.590074:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.590078:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019269.590083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019269.590094:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335675264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019269.590103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019269.590106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019269.590107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.590112:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.590114:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.590119:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019269.590123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019269.590124:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335675008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019269.590132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335675392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019269.590141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019269.590144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019269.590145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.590149:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.590152:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.590155:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019269.590159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019269.590167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335675520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019269.590175:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019269.590179:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019269.590180:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.590184:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.590187:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.590190:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019269.590194:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019269.590202:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335675648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019269.590209:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734019269.590210:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019269.590213:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019269.590215:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019269.590219:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3534 00000100:00080000:0.0:1734019269.590222:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019269.590225:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019269.590229:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019269.590237:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335675776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019269.590239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59ca300 x1818247335675008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335675008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590272:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590281:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335675008:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019269.590288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335675136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019269.590309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019269.590316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59caa00 x1818247335675136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335675136/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590342:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335675136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019269.590348:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335675264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019269.590362:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019269.590368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59cb100 x1818247335675264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590378:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335675264/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590387:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335675264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019269.590420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335675392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019269.590433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019269.590440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59c8000 x1818247335675392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335675392/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335675392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019269.590475:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335675520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019269.590488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019269.590493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59c8e00 x1818247335675520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335675520/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590511:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590513:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590516:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335675520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019269.590521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335675648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019269.590533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019269.590539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59c8a80 x1818247335675648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335675648/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590563:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335675648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019269.590568:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335675776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019269.590581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019269.590586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019269/real 1734019269] req@ffff8800b59cb480 x1818247335675776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019269.590596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335675776/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019269.590603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019269.590605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019269.590608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335675776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019270.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019270.507968:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019271.213894:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019271.509988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019272.213880:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019272.512020:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019273.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019273.513992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019274.213888:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019274.516014:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019274.597847:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.597858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019274.597863:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.597874:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.597883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.597892:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019274.597929:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335675904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019274.597963:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.597967:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019274.597969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.597974:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.597977:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.597981:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019274.597990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019274.598001:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335676032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019274.598013:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.598016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019274.598017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.598021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.598023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.598027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019274.598031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019274.598039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335676160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019274.598048:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.598051:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019274.598053:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.598056:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.598058:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.598062:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019274.598066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019274.598073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335676288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019274.598083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.598086:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019274.598086:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335675904:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019274.598087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.598091:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.598095:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.598098:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019274.598102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019274.598110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335676416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019274.598117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.598120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019274.598122:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.598125:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.598127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.598130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019274.598134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019274.598142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335676544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019274.598149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019274.598152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019274.598154:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019274.598157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3539 00000100:00080000:0.0:1734019274.598160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019274.598163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019274.598167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019274.598175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335676672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019274.598176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59c9c00 x1818247335675904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598228:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335675904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598241:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335675904:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019274.598259:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019274.598278:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598285:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59cad80 x1818247335676032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598296:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335676032/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598304:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019274.598318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019274.598332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598338:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59c8700 x1818247335676160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335676160/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598362:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019274.598395:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019274.598412:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598418:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59cb800 x1818247335676288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335676288/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019274.598449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335676416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019274.598461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59c9500 x1818247335676416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335676416/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598487:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335676416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019274.598495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335676544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019274.598508:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59ca680 x1818247335676544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335676544/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335676544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019274.598551:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335676672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019274.598563:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019274.598569:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019274/real 1734019274] req@ffff8800b59cbb80 x1818247335676672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019274.598579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335676672/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019274.598586:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019274.598589:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019274.598592:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335676672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019275.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019275.517985:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019276.213892:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019276.520041:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019277.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019277.522008:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019278.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019278.523987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019279.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019279.525987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019279.605832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.605842:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019279.605848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.605861:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.605872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.605882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019279.605931:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335676800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019279.605969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.605973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019279.605975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.605980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.605983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.605987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019279.605998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019279.606010:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335676928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019279.606020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.606023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019279.606025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.606029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.606032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.606036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019279.606042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019279.606054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335677056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019279.606064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.606067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019279.606069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.606073:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.606075:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.606079:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335676800:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019279.606083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019279.606092:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335677184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019279.606100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.606103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019279.606105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.606109:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.606112:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.606116:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019279.606120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019279.606128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335677312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019279.606136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.606139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019279.606141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.606145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.606148:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.606152:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019279.606156:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019279.606161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019279.606164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335677440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019279.606172:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019279.606175:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019279.606176:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019279.606179:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3544 00000100:00080000:0.0:1734019279.606182:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019279.606185:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019279.606189:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019279.606190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59c9180 x1818247335676800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019279.606198:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335677568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019279.606212:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335676800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606224:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606227:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335676800:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019279.606240:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019279.606257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606264:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59c9f80 x1818247335676928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019279.606275:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335676928/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335676928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019279.606318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019279.606332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59cbb80 x1818247335677056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019279.606349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335677056/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019279.606370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019279.606382:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59ca680 x1818247335677184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019279.606397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335677184/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606405:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606411:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019279.606416:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335677312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019279.606428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59c9500 x1818247335677312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019279.606444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335677312/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606453:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606456:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335677312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019279.606461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335677440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019279.606477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59cb800 x1818247335677440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019279.606492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335677440/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335677440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019279.606510:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335677568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019279.606522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019279.606528:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019279/real 1734019279] req@ffff8800b59c8700 x1818247335677568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019279.606537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335677568/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019279.606544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019279.606546:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019279.606549:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335677568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019280.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019280.527971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019281.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019281.529982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019282.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019282.531969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019283.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019283.533972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019284.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019284.536010:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019284.613843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019284.613852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019284.613857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.613870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.613880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.613890:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019284.613937:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335677696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019284.613982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019284.613986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019284.613988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.613993:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.613997:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.614000:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019284.614012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019284.614024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335677824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019284.614034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019284.614037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019284.614039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.614043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.614045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.614049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019284.614054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019284.614062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335677952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019284.614070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019284.614073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019284.614075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.614078:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.614081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.614085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019284.614090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019284.614095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335677696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019284.614098:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335678080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019284.614108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019284.614111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019284.614113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.614117:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.614119:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.614123:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019284.614127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019284.614134:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335678208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019284.614142:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019284.614145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019284.614147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.614150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.614153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.614157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019284.614161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019284.614169:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335678336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019284.614177:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734019284.614179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019284.614181:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019284.614183:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019284.614187:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3549 00000100:00080000:0.0:1734019284.614190:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019284.614194:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019284.614198:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019284.614207:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59cad80 x1818247335677696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019284.614209:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335678464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019284.614229:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335677696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614249:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335677696:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019284.614256:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019284.614273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019284.614280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59c9c00 x1818247335677824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019284.614292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335677824/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614302:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019284.614311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019284.614325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019284.614331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59cb480 x1818247335677952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019284.614341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335677952/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335677952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019284.614386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019284.614398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019284.614404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59c8a80 x1818247335678080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019284.614414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335678080/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019284.614433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335678208:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019284.614444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019284.614450:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59c8e00 x1818247335678208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019284.614460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335678208/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614473:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335678208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019284.614478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335678336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019284.614490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019284.614497:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59c8000 x1818247335678336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019284.614507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335678336/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614514:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614516:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614519:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335678336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019284.614524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335678464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019284.614536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019284.614542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019284/real 1734019284] req@ffff8800b59cb100 x1818247335678464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019284.614552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335678464/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019284.614559:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019284.614561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019284.614564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335678464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019285.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019285.537974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019286.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019286.539997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019287.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019287.542010:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019288.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019288.543991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019289.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019289.545980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019289.621832:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.621842:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019289.621846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.621857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00080000:0.0:1734019289.621865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.621873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019289.621907:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335678592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019289.621942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.621946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019289.621948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.621953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00080000:0.0:1734019289.621956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.621959:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019289.621969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019289.621979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335678720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019289.621990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.621993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019289.621994:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.621998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00080000:0.0:1734019289.622000:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.622003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019289.622008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019289.622017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335678848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019289.622026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.622029:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019289.622030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.622034:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00080000:0.0:1734019289.622036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.622040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019289.622043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019289.622051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335678976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019289.622061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.622064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019289.622065:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.622069:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00100000:3.0:1734019289.622071:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335678592:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019289.622072:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.622076:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019289.622081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019289.622088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335679104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019289.622096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.622099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019289.622100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.622103:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00080000:0.0:1734019289.622106:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.622109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019289.622113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019289.622120:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335679232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019289.622128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019289.622131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019289.622132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019289.622135:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3554 00000100:00080000:0.0:1734019289.622140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019289.622143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019289.622147:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019289.622154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335679360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019289.622169:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622201:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59caa00 x1818247335678592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335678592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622237:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335678592:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019289.622255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019289.622273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59ca300 x1818247335678720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335678720/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622300:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019289.622313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019289.622327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59cb100 x1818247335678848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622345:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335678848/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019289.622391:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019289.622405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59c8000 x1818247335678976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335678976/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335678976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019289.622442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335679104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019289.622454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59c8e00 x1818247335679104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335679104/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622478:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622481:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622484:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335679104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019289.622491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335679232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019289.622504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59c8a80 x1818247335679232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335679232/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335679232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019289.622601:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335679360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019289.622620:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019289.622628:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019289/real 1734019289] req@ffff8800b59cb480 x1818247335679360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019289.622638:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335679360/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019289.622646:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019289.622648:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019289.622651:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335679360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019290.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019290.547948:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019291.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019291.549926:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019292.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019292.551993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019293.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019293.554009:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019294.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019294.555984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019294.629814:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.629823:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019294.629828:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.629839:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.629848:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.629855:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019294.629892:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335679488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019294.629926:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.629930:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019294.629932:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.629936:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.629939:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.629943:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019294.629953:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019294.629964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335679616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019294.629973:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.629977:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019294.629978:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.629982:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.629985:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.629988:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019294.629994:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019294.630002:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335679744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019294.630011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.630014:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019294.630015:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.630018:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.630021:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.630024:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019294.630028:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019294.630036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335679872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019294.630045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.630048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019294.630048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335679488:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019294.630050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.630053:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.630056:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.630059:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019294.630063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019294.630071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335680000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019294.630079:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.630082:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019294.630084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.630088:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.630090:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.630093:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019294.630097:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019294.630104:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335680128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019294.630112:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019294.630115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019294.630116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019294.630119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3559 00000100:00080000:0.0:1734019294.630122:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019294.630125:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019294.630129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019294.630137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335680256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019294.630166:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335679616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019294.630190:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335679744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019294.630219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335679872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019294.630237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019294.630255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019294.630271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019295.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019295.557971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019296.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019296.560001:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019297.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019297.561954:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734019297.635991:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019297.636042:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019297.636064:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019297.636102:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019297.636112:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019297.636120:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019297.636128:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019297.636135:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019297.636143:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019297.636174:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59cb800 x1818247335679872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335679872/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335679872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019297.636261:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59c9500 x1818247335680000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636283:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335680000/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636296:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019297.636312:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59ca680 x1818247335680128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636331:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335680128/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019297.636365:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59cbb80 x1818247335680256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335680256/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019297.636474:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59c9c00 x1818247335679488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335679488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636534:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335679488:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019297.636551:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59cad80 x1818247335679616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335679616/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636583:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636587:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636592:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335679616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019297.636603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019294/real 1734019297] req@ffff8800b59c8700 x1818247335679744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019297.636622:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335679744/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019297.636634:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019297.636638:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019297.636654:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335679744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019298.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019298.563964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019299.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019299.565938:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019299.637827:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.637836:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019299.637841:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.637855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.637866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.637875:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019299.637924:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335680384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019299.637968:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.637973:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019299.637975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.637980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.637983:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.637987:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019299.637998:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019299.638009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335680512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019299.638018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.638022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019299.638023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.638027:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.638030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.638034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019299.638039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019299.638049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335680640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019299.638058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.638061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019299.638062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.638066:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.638068:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.638073:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019299.638077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019299.638079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335680384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019299.638086:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335680768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019299.638095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.638098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019299.638100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.638103:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.638106:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.638109:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019299.638113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019299.638121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335680896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019299.638129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.638132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019299.638133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.638137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.638140:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.638143:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019299.638148:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019299.638156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335681024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019299.638162:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019299.638163:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019299.638167:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019299.638169:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019299.638173:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3564 00000100:00080000:0.0:1734019299.638176:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019299.638179:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019299.638183:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019299.638190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59c9f80 x1818247335680384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019299.638192:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335681152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019299.638211:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335680384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638227:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638232:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335680384:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019299.638239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335680512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019299.638256:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019299.638263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59c9180 x1818247335680512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019299.638275:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335680512/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638289:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335680512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019299.638295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335680640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019299.638309:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019299.638315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59c8700 x1818247335680640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019299.638333:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335680640/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638341:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335680640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019299.638420:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335680768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019299.638437:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019299.638445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59cad80 x1818247335680768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019299.638455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335680768/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335680768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019299.638474:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019299.638487:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019299.638493:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59c9c00 x1818247335680896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019299.638503:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335680896/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335680896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019299.638524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019299.638536:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019299.638542:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59cbb80 x1818247335681024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019299.638552:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335681024/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638560:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638562:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638565:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019299.638570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019299.638582:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019299.638588:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019299/real 1734019299] req@ffff8800b59ca680 x1818247335681152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019299.638597:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335681152/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019299.638604:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019299.638607:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019299.638610:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019300.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019300.568004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019301.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019301.569951:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019302.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019302.571958:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019303.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019303.573986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019304.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019304.575967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019304.645823:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.645832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019304.645837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.645846:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00080000:0.0:1734019304.645855:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.645863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019304.645898:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335681280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019304.645931:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.645935:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019304.645937:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.645942:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00080000:0.0:1734019304.645945:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.645948:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019304.645958:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019304.645968:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335681408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019304.645979:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.645982:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019304.645983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.645987:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00080000:0.0:1734019304.645989:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.645992:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019304.645996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019304.646004:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335681536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019304.646013:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.646016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019304.646017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.646021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00080000:0.0:1734019304.646023:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.646027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019304.646031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019304.646038:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335681664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019304.646047:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.646050:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019304.646052:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.646056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00100000:3.0:1734019304.646056:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335681280:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019304.646059:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.646062:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019304.646066:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019304.646074:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335681792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019304.646082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.646085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019304.646086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.646089:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00080000:0.0:1734019304.646091:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.646095:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019304.646098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019304.646105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335681920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019304.646113:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019304.646116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019304.646117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019304.646121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3569 00000100:00080000:0.0:1734019304.646124:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019304.646127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019304.646130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019304.646138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335682048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019304.646150:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646182:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59c9500 x1818247335681280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646204:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335681280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646218:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646227:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335681280:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019304.646234:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335681408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019304.646253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646260:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59cb800 x1818247335681408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335681408/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646326:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646329:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646333:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335681408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019304.646340:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335681536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019304.646359:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646368:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59cb480 x1818247335681536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646379:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335681536/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646392:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335681536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019304.646425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335681664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019304.646439:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59c8a80 x1818247335681664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646455:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335681664/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335681664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019304.646476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019304.646488:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646494:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59c8e00 x1818247335681792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335681792/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646522:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646525:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019304.646531:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019304.646543:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646549:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59c8000 x1818247335681920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335681920/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646569:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646572:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335681920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019304.646578:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019304.646591:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019304.646597:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019304/real 1734019304] req@ffff8800b59cb100 x1818247335682048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019304.646607:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335682048/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019304.646614:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019304.646616:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019304.646619:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019305.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019305.577931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019306.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019306.579977:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019307.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019307.581983:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019308.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019308.583951:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019309.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019309.585987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019309.653819:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.653828:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019309.653833:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019309.653842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.653851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.653859:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019309.653895:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335682176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019309.653929:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.653933:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019309.653935:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019309.653939:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.653942:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.653946:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019309.653956:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019309.653967:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335682304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019309.653976:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.653980:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019309.653981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019309.653985:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.653988:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.653991:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019309.653996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019309.654006:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335682432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019309.654015:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.654018:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019309.654020:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019309.654023:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.654025:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.654029:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019309.654032:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019309.654040:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335682560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019309.654049:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.654052:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019309.654054:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019309.654055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335682176:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019309.654058:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.654061:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.654064:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019309.654069:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019309.654076:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335682688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019309.654084:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.654087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019309.654089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019309.654093:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.654095:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.654098:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019309.654102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019309.654110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335682816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019309.654117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019309.654120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019309.654122:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019309.654125:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3574 00000100:00080000:0.0:1734019309.654128:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019309.654131:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019309.654136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019309.654142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335682944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019309.654147:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654178:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59ca300 x1818247335682176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335682176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654244:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654248:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335682176:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019309.654262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335682304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019309.654284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59caa00 x1818247335682304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335682304/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335682304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019309.654329:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335682432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019309.654345:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59cb100 x1818247335682432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335682432/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335682432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019309.654407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335682560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019309.654420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59c8000 x1818247335682560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335682560/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335682560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019309.654457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019309.654470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59c8e00 x1818247335682688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335682688/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019309.654506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019309.654518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59c8a80 x1818247335682816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335682816/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654541:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654544:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019309.654552:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019309.654566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019309.654572:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019309/real 1734019309] req@ffff8800b59cb480 x1818247335682944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019309.654583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335682944/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019309.654590:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019309.654593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019309.654596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335682944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019310.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019310.587955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019311.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019311.589982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019312.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019312.592003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019313.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019313.593971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019314.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019314.595976:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019314.661812:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.661822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019314.661826:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.661836:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00080000:0.0:1734019314.661845:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.661853:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019314.661887:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335683072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019314.661928:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.661932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019314.661934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.661938:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00080000:0.0:1734019314.661941:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.661945:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019314.661954:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019314.661964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335683200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019314.661974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.661977:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019314.661978:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.661982:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00080000:0.0:1734019314.661984:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.661987:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019314.661992:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019314.661999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335683328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019314.662008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.662011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019314.662012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.662016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00080000:0.0:1734019314.662018:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.662022:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019314.662026:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019314.662034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335683456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019314.662043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.662046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019314.662048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.662051:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00100000:3.0:1734019314.662053:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335683072:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019314.662054:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.662057:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019314.662061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019314.662068:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335683584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019314.662076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.662079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019314.662081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.662084:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00080000:0.0:1734019314.662086:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.662089:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019314.662094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019314.662101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335683712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019314.662140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019314.662144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019314.662146:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019314.662150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3579 00000100:00080000:0.0:1734019314.662153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019314.662156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019314.662160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019314.662170:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335683840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019314.662183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59cb800 x1818247335683072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662234:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335683072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335683072:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019314.662266:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335683200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019314.662286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59c9500 x1818247335683200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662305:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335683200/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662313:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335683200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019314.662327:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335683328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019314.662342:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59ca680 x1818247335683328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335683328/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335683328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019314.662404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335683456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019314.662417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59cbb80 x1818247335683456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335683456/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662442:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335683456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019314.662453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335683584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019314.662466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662472:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59c9c00 x1818247335683584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335683584/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335683584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019314.662503:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335683712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019314.662515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59cad80 x1818247335683712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335683712/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335683712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019314.662549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335683840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019314.662562:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019314.662568:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019314/real 1734019314] req@ffff8800b59c8700 x1818247335683840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019314.662579:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335683840/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019314.662594:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019314.662597:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019314.662600:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335683840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019315.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019315.597949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019316.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019316.599997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019317.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019317.601986:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019318.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019318.603957:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019319.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019319.605970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019319.669842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.669851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019319.669855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.669865:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.669874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019319.669882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019319.669917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335683968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019319.669950:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.669954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019319.669956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.669960:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.669963:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019319.669967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019319.669977:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019319.669987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335684096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019319.669998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.670001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019319.670002:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.670006:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.670008:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019319.670012:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019319.670016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019319.670066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335684224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019319.670077:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.670080:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019319.670081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.670085:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.670087:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019319.670091:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019319.670095:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019319.670097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335683968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019319.670104:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335684352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019319.670112:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.670115:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019319.670117:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.670121:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.670124:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019319.670127:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019319.670132:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019319.670139:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335684480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019319.670147:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.670150:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019319.670151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.670155:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.670157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019319.670160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019319.670164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019319.670171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335684608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019319.670178:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019319.670181:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019319.670183:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019319.670186:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3584 00000100:00080000:0.0:1734019319.670189:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734019319.670190:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019319.670192:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019319.670196:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019319.670204:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335684736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019319.670220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59c9180 x1818247335683968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335683968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335683968:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019319.670273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019319.670293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019319.670299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59c9f80 x1818247335684096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335684096/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670328:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019319.670363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019319.670378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019319.670385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59c8700 x1818247335684224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670395:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335684224/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019319.670415:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019319.670428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019319.670434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59cad80 x1818247335684352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335684352/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019319.670464:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335684480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019319.670479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019319.670485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59c9c00 x1818247335684480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335684480/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335684480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019319.670514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335684608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019319.670527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019319.670532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59cbb80 x1818247335684608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335684608/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670552:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670555:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335684608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019319.670561:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335684736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019319.670575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019319.670581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019319/real 1734019319] req@ffff8800b59ca680 x1818247335684736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019319.670591:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335684736/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019319.670598:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019319.670601:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019319.670604:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335684736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019320.213918:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019320.607980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019321.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019321.609967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019322.213801:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019322.612023:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019323.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019323.614025:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019324.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019324.615923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019324.677831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019324.677841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019324.677846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019324.677855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.677865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.677873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019324.677908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335684864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019324.678115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019324.678120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019324.678123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019324.678128:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.678132:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.678135:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019324.678145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019324.678147:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335684864:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019324.678158:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335684992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019324.678170:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019324.678173:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019324.678176:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019324.678180:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.678182:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.678185:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019324.678190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019324.678197:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335685120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019324.678206:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019324.678209:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019324.678210:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019324.678213:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.678216:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.678220:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019324.678222:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734019324.678224:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019324.678231:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335685248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019324.678240:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019324.678243:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019324.678245:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019324.678246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59c9500 x1818247335684864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019324.678249:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.678252:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.678255:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019324.678259:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019324.678260:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335684864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019324.678267:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335685376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019324.678269:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019324.678275:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335684864:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019324.678276:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019324.678279:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019324.678281:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019324.678284:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.678286:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.678290:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019324.678294:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019324.678297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:0.0:1734019324.678302:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335685504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019324.678310:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734019324.678310:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019324.678313:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019324.678315:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019324.678315:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59cb800 x1818247335684992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019324.678318:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3589 00000100:00080000:0.0:1734019324.678321:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019324.678324:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019324.678328:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019324.678328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335684992/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019324.678334:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678335:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1734019324.678338:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335685632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019324.678338:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335684992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019324.678343:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335685120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019324.678354:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019324.678358:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59cb480 x1818247335685120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019324.678365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335685120/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019324.678369:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678371:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019324.678373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335685120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019324.678377:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335685248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019324.678387:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019324.678391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59c8a80 x1818247335685248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019324.678397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335685248/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019324.678402:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678405:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019324.678407:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335685248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019324.678412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335685376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019324.678421:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019324.678425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59c8e00 x1818247335685376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019324.678430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335685376/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019324.678434:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019324.678438:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335685376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019324.678453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335685504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019324.678461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019324.678464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59c8000 x1818247335685504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019324.678470:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335685504/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019324.678476:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019324.678479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335685504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019324.678483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335685632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019324.678493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019324.678497:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019324/real 1734019324] req@ffff8800b59cb100 x1818247335685632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019324.678504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335685632/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019324.678508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019324.678510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019324.678511:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335685632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019325.213961:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019325.617984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019326.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019326.619982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019327.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019327.622007:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019328.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019328.624005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019329.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019329.625992:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019329.685842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.685890:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019329.685894:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.685904:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.685913:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019329.685922:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019329.685957:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335685760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019329.685990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.685995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019329.685997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.686001:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.686004:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019329.686008:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019329.686018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019329.686028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335685888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019329.686038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.686041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019329.686043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.686047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.686049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019329.686052:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019329.686057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019329.686066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335686016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019329.686075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.686078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019329.686080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.686084:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.686087:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019329.686090:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019329.686094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019329.686102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335686144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019329.686111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.686114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019329.686116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.686120:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.686122:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019329.686123:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335685760:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019329.686126:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019329.686130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019329.686138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335686272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019329.686146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.686149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019329.686150:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.686154:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.686156:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019329.686160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019329.686164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019329.686171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335686400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019329.686179:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019329.686182:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019329.686184:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019329.686187:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3594 00000100:00080000:0.0:1734019329.686190:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019329.686193:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019329.686197:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019329.686204:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335686528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019329.686219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59caa00 x1818247335685760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686272:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335685760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686284:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335685760:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019329.686301:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335685888:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019329.686324:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59ca300 x1818247335685888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335685888/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335685888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019329.686364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019329.686380:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686386:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59cb100 x1818247335686016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335686016/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686403:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019329.686440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019329.686454:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686460:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59c8000 x1818247335686144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335686144/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019329.686498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335686272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019329.686511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59c8e00 x1818247335686272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335686272/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335686272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019329.686546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335686400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019329.686559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59c8a80 x1818247335686400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335686400/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335686400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019329.686595:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335686528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019329.686609:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019329.686615:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019329/real 1734019329] req@ffff8800b59cb480 x1818247335686528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019329.686625:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335686528/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019329.686632:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019329.686635:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019329.686638:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335686528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019330.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019330.628053:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019331.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019331.629997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019332.213856:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019332.632028:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019333.213877:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019333.634043:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019334.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019334.635966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019334.693844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.693854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019334.693858:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.693868:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.693877:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.693884:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019334.693920:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335686656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019334.693953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.693957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019334.693959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.693964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.693967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.693970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019334.693981:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019334.693991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335686784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019334.694001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.694004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019334.694006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.694010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.694012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.694015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019334.694019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019334.694027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335686912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019334.694036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.694039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019334.694040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.694044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.694046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.694049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019334.694053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019334.694060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335687040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019334.694070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.694073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019334.694075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.694079:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.694081:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.694084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694085:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335686656:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019334.694088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019334.694096:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335687168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019334.694103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.694106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019334.694107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.694111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.694113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.694116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019334.694120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019334.694127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335687296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019334.694135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019334.694138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019334.694139:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019334.694143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3599 00000100:00080000:0.0:1734019334.694145:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019334.694148:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019334.694152:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019334.694159:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335687424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019334.694179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59cb800 x1818247335686656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694230:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335686656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694253:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335686656:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019334.694261:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019334.694280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694287:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59c9500 x1818247335686784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335686784/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694313:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019334.694320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019334.694335:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59ca680 x1818247335686912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335686912/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335686912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019334.694399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019334.694417:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59cbb80 x1818247335687040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694435:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335687040/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694443:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694445:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019334.694454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335687168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019334.694467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59c9c00 x1818247335687168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335687168/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694492:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335687168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019334.694501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335687296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019334.694514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59cad80 x1818247335687296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694537:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335687296/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694544:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694547:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694550:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335687296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019334.694555:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335687424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019334.694568:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019334.694574:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019334/real 1734019334] req@ffff8800b59c8700 x1818247335687424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019334.694584:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335687424/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019334.694591:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019334.694593:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019334.694596:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335687424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019335.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019335.637995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019336.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019336.639987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019337.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019337.641974:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019338.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019338.643892:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019339.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019339.646012:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019339.701847:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.701856:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019339.701860:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019339.701870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.701879:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.701887:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019339.701922:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335687552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019339.701956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.701960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019339.701962:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019339.701967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.701969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.701973:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019339.701982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019339.701993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335687680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019339.702002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.702006:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019339.702007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019339.702011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.702014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.702017:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019339.702022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019339.702032:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335687808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019339.702041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.702044:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019339.702046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019339.702049:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.702052:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.702055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019339.702059:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019339.702066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335687936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019339.702076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.702079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019339.702080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019339.702083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335687552:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019339.702084:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.702086:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.702090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019339.702094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019339.702102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335688064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019339.702110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.702113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019339.702115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019339.702118:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.702121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.702124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019339.702128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019339.702135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335688192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019339.702143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019339.702145:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019339.702147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019339.702150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3604 00000100:00080000:0.0:1734019339.702153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019339.702156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019339.702160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019339.702168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335688320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019339.702179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59c9f80 x1818247335687552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335687552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702249:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702255:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335687552:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019339.702263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019339.702282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59c9180 x1818247335687680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335687680/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019339.702321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019339.702337:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59c8700 x1818247335687808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335687808/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702360:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019339.702400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019339.702414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702420:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59cad80 x1818247335687936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335687936/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702439:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335687936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019339.702450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019339.702463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59c9c00 x1818247335688064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335688064/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019339.702498:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019339.702515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59cbb80 x1818247335688192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702532:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335688192/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702540:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702542:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702545:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019339.702551:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019339.702564:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019339.702570:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019339/real 1734019339] req@ffff8800b59ca680 x1818247335688320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019339.702580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335688320/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019339.702587:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019339.702589:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019339.702592:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019340.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019340.647964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019341.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019341.649962:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019342.213802:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019342.651973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019343.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019343.653987:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019344.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019344.655933:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019344.709843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.709853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019344.709857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019344.709868:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.709876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.709884:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019344.709920:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335688448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019344.709963:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.709967:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019344.709968:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019344.709973:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.709976:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.709980:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019344.709989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019344.709999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335688576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019344.710008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.710012:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019344.710013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019344.710017:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.710019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.710023:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019344.710027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019344.710036:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335688704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019344.710044:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.710047:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019344.710049:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019344.710052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.710055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.710059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019344.710063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019344.710071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335688832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019344.710080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.710083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019344.710085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019344.710088:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335688448:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019344.710089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.710091:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.710095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019344.710099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019344.710107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335688960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019344.710115:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.710118:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019344.710119:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019344.710123:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.710125:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.710128:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019344.710133:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019344.710140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335689088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019344.710148:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019344.710152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019344.710154:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019344.710157:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3609 00000100:00080000:0.0:1734019344.710160:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019344.710163:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019344.710167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019344.710176:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335689216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019344.710183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59c9500 x1818247335688448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335688448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335688448:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019344.710265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335688576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019344.710284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59cb800 x1818247335688576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335688576/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710317:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335688576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019344.710324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335688704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019344.710338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59cb480 x1818247335688704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335688704/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335688704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019344.710401:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335688832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019344.710415:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710421:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59c8a80 x1818247335688832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710431:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335688832/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710440:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710442:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710445:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335688832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019344.710451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019344.710463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59c8e00 x1818247335688960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335688960/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710489:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710492:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335688960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019344.710499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019344.710511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59c8000 x1818247335689088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335689088/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710535:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019344.710546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019344.710559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019344.710565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019344/real 1734019344] req@ffff8800b59cb100 x1818247335689216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019344.710575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335689216/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019344.710584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019344.710586:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019344.710589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019345.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019345.657950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019346.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019346.659956:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019347.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019347.662006:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019348.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019348.663944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019349.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019349.665913:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019349.717805:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.717814:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019349.717818:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.717828:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.717837:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.717844:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019349.717878:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335689344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019349.717912:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.717917:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019349.717919:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.717923:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.717926:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.717930:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019349.717939:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019349.717949:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335689472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019349.717960:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.717963:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019349.717964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.717968:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.717970:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.717973:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019349.717978:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019349.717988:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335689600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019349.717996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.717999:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019349.718001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.718004:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.718007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.718010:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019349.718014:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019349.718022:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335689728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019349.718032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.718035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019349.718035:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335689344:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019349.718036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.718040:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.718043:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.718047:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019349.718051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019349.718059:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335689856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019349.718067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.718069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019349.718071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.718074:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.718076:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.718080:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019349.718084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019349.718090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335689984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019349.718098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019349.718101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019349.718102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019349.718105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3614 00000100:00080000:0.0:1734019349.718108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019349.718111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019349.718115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019349.718122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335690112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019349.718129:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718159:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59ca300 x1818247335689344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718181:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335689344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718194:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718204:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335689344:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019349.718211:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335689472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019349.718230:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59caa00 x1818247335689472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335689472/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335689472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019349.718270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335689600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019349.718284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59cb100 x1818247335689600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335689600/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335689600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019349.718347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335689728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019349.718361:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59c8000 x1818247335689728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718377:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335689728/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718385:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718387:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718391:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335689728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019349.718397:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689856:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019349.718410:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718415:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59c8e00 x1818247335689856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718425:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335689856/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718433:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718436:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718439:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019349.718445:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019349.718457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718463:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59c8a80 x1818247335689984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718473:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335689984/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335689984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019349.718491:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335690112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019349.718504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019349.718555:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019349/real 1734019349] req@ffff8800b59cb480 x1818247335690112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019349.718566:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335690112/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019349.718573:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019349.718575:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019349.718579:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335690112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019350.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019350.667975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019351.213860:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019351.669969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019352.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019352.672003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019353.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019353.673980:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019354.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019354.675984:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019354.725813:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.725822:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019354.725826:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.725835:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.725844:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.725852:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019354.725886:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335690240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019354.725920:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.725924:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019354.725926:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.725930:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.725933:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.725937:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019354.725946:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019354.725956:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335690368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019354.725966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.725969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019354.725971:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.725975:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.725977:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.725981:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019354.725986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019354.725994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335690496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019354.726002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.726005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019354.726007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.726010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.726012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.726016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019354.726020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019354.726027:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335690624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019354.726036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.726039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019354.726040:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335690240:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019354.726041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.726045:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.726047:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.726050:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019354.726054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019354.726062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335690752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019354.726070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.726073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019354.726075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.726078:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.726081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.726084:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019354.726088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019354.726095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335690880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019354.726103:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019354.726105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019354.726107:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019354.726110:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3619 00000100:00080000:0.0:1734019354.726113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019354.726116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019354.726120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019354.726129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335691008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019354.726131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726161:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59cb800 x1818247335690240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726183:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335690240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726195:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726200:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726205:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335690240:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019354.726213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335690368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019354.726231:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726238:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59c9500 x1818247335690368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335690368/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335690368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019354.726271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335690496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019354.726286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59ca680 x1818247335690496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335690496/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335690496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019354.726350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335690624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019354.726364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726370:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59cbb80 x1818247335690624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726380:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335690624/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726387:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726390:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726393:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335690624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019354.726398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335690752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019354.726411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59c9c00 x1818247335690752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335690752/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726497:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726500:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335690752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019354.726506:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335690880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019354.726522:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59cad80 x1818247335690880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335690880/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335690880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019354.726559:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019354.726572:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019354.726578:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019354/real 1734019354] req@ffff8800b59c8700 x1818247335691008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019354.726588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335691008/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019354.726595:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019354.726598:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019354.726601:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019355.213924:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019355.677989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019356.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019356.679979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019357.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019357.682007:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019358.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019358.683978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019359.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019359.685988:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019359.733835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.733845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019359.733850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.733860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.733868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019359.733876:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019359.733912:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335691136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019359.733947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.733951:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019359.733953:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.733957:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.733960:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019359.733964:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019359.733973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019359.733984:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335691264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019359.733994:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.733997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019359.733999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.734002:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.734005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019359.734008:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019359.734013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019359.734022:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335691392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019359.734031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.734034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019359.734036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.734039:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.734042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019359.734046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019359.734050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019359.734058:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335691520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019359.734067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.734070:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019359.734072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.734075:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.734078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019359.734079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335691136:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019359.734081:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019359.734085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019359.734093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335691648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019359.734100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.734103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019359.734105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.734108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.734110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019359.734114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019359.734118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019359.734125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335691776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019359.734133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019359.734137:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019359.734138:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019359.734141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3624 00000100:00080000:0.0:1734019359.734146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019359.734149:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019359.734153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019359.734160:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335691904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019359.734209:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335691264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019359.734233:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335691392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019359.734262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335691520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019359.734281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019359.734298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019359.734316:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019360.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019360.687979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019361.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019361.689933:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019362.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019362.691963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734019362.739980:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019362.740027:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019362.740047:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019362.740083:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019362.740094:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019362.740102:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019362.740110:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019362.740117:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019362.740124:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019362.740197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59cad80 x1818247335691520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335691520/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740252:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335691520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019362.740268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59c9c00 x1818247335691648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335691648/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740346:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019362.740354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59cbb80 x1818247335691776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335691776/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019362.740389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59ca680 x1818247335691904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335691904/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740407:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740409:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335691904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019362.740423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59c9180 x1818247335691136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335691136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335691136:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019362.740457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59c9f80 x1818247335691264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335691264/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335691264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019362.740488:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019359/real 1734019362] req@ffff8800b59c8700 x1818247335691392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019362.740499:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335691392/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019362.740506:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019362.740509:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019362.740514:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335691392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019363.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019363.693986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019364.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019364.695975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019364.741823:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.741833:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019364.741837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.741847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.741856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.741863:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019364.741898:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335692032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019364.741934:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.741938:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019364.741940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.741945:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.741948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.741951:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019364.741960:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019364.741971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335692160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019364.741982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.741985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019364.741986:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.741990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.741992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.741996:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019364.742000:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019364.742008:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335692288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019364.742017:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.742020:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019364.742022:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.742025:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.742027:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.742031:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019364.742035:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019364.742043:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335692416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019364.742052:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.742055:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019364.742055:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335692032:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019364.742057:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.742061:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.742064:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.742067:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019364.742072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019364.742079:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335692544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019364.742087:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.742090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019364.742092:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.742095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.742097:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.742100:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019364.742104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019364.742113:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335692672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019364.742121:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019364.742124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019364.742125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019364.742128:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3629 00000100:00080000:0.0:1734019364.742131:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019364.742134:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019364.742138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019364.742146:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335692800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019364.742151:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742181:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59c9500 x1818247335692032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742202:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335692032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742215:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742219:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742224:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335692032:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019364.742232:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335692160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019364.742292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59cb800 x1818247335692160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335692160/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742330:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335692160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019364.742337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335692288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019364.742356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59cb480 x1818247335692288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335692288/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742384:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742387:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335692288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019364.742417:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335692416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019364.742430:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59c8a80 x1818247335692416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335692416/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742454:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742457:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742460:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335692416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019364.742466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335692544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019364.742479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59c8e00 x1818247335692544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335692544/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742502:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335692544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019364.742515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335692672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019364.742527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59c8000 x1818247335692672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335692672/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335692672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019364.742561:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335692800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019364.742574:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019364.742580:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019364/real 1734019364] req@ffff8800b59cb100 x1818247335692800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019364.742590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335692800/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019364.742597:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019364.742599:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019364.742602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335692800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019365.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019365.697979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019366.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019366.699945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019367.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019367.701995:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019368.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019368.703975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019369.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019369.705974:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019369.749803:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.749812:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019369.749817:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019369.749827:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.749835:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.749844:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019369.749878:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335692928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019369.749913:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.749917:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019369.749919:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019369.749923:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.749926:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.749930:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019369.749940:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019369.749950:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335693056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019369.749960:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.749963:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019369.749965:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019369.749969:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.749971:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.749974:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019369.749980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019369.749991:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335693184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019369.750000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.750003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019369.750004:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019369.750008:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.750010:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.750014:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019369.750018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019369.750025:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335693312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019369.750035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.750038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019369.750039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019369.750040:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335692928:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019369.750043:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.750046:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.750050:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019369.750054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019369.750062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335693440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019369.750070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.750073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019369.750075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019369.750079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.750082:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.750085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019369.750089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019369.750097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335693568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019369.750104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019369.750107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019369.750109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019369.750112:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3634 00000100:00080000:0.0:1734019369.750115:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019369.750118:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019369.750122:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019369.750130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335693696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019369.750131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750162:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59caa00 x1818247335692928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335692928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335692928:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019369.750258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019369.750281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59ca300 x1818247335693056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335693056/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019369.750321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019369.750336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59cb100 x1818247335693184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335693184/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019369.750400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019369.750414:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750420:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59c8000 x1818247335693312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335693312/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750441:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019369.750451:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335693440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019369.750463:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750469:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59c8e00 x1818247335693440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750479:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335693440/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750498:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750501:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335693440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019369.750507:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335693568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019369.750518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59c8a80 x1818247335693568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750534:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335693568/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750544:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335693568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019369.750553:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335693696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019369.750566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019369.750573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019369/real 1734019369] req@ffff8800b59cb480 x1818247335693696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019369.750582:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335693696/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019369.750589:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019369.750592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019369.750595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335693696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019370.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019370.707971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019371.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019371.709940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019372.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019372.711906:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019373.213899:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019373.713941:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019374.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019374.715902:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019374.757773:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.757781:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019374.757785:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.757795:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.757803:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.757809:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019374.757838:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335693824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019374.757864:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.757867:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019374.757869:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.757872:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.757874:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.757877:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019374.757884:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019374.757892:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335693952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019374.757899:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.757901:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019374.757902:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.757905:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.757907:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.757909:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019374.757913:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019374.757918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335694080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019374.757924:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.757926:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019374.757927:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.757930:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.757931:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.757934:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019374.757937:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019374.757943:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335694208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019374.757949:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.757951:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019374.757952:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.757955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.757956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.757959:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019374.757961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019374.757967:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335694336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019374.757975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.757977:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019374.757978:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.757981:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.757982:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.757985:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019374.757988:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019374.757994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335694464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019374.757997:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335693824:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019374.758000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019374.758002:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019374.758004:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019374.758006:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3639 00000100:00080000:0.0:1734019374.758008:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019374.758010:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019374.758013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019374.758019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335694592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019374.758135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758164:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59cb800 x1818247335693824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335693824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758198:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758202:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758207:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335693824:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019374.758216:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019374.758238:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59c9500 x1818247335693952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758256:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335693952/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758265:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335693952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019374.758281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019374.758295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758301:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59c8700 x1818247335694080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758312:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335694080/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758320:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019374.758331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019374.758343:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758349:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59c9f80 x1818247335694208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758358:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335694208/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758366:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758371:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019374.758405:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335694336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019374.758419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59c9180 x1818247335694336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335694336/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758450:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335694336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019374.758456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335694464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019374.758467:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59ca680 x1818247335694464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758483:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335694464/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758490:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758495:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335694464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019374.758501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335694592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019374.758514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019374.758520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019374/real 1734019374] req@ffff8800b59cbb80 x1818247335694592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019374.758530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335694592/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019374.758537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019374.758540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019374.758543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335694592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019375.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019375.718033:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019376.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019376.719963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019377.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019377.722001:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019378.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019378.723959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019379.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019379.726051:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019379.765809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.765819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019379.765824:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.765834:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.765843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.765851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019379.765888:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335694720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019379.765923:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.765928:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019379.765930:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.765934:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.765938:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.765942:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019379.765952:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019379.765963:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335694848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019379.765973:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.765976:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019379.765977:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.765981:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.765983:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.765987:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019379.765991:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019379.766031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335694976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019379.766041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.766044:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019379.766045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.766049:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.766051:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.766055:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019379.766060:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019379.766069:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335695104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019379.766070:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335694720:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019379.766085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.766089:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019379.766090:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.766094:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.766096:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.766100:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019379.766104:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019379.766111:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335695232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019379.766119:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.766122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019379.766123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.766126:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.766129:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.766132:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019379.766136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019379.766143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335695360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019379.766151:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019379.766154:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019379.766155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019379.766159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3644 00000100:00080000:0.0:1734019379.766162:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019379.766165:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019379.766165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:0.0:1734019379.766170:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019379.766178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335695488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019379.766196:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59c9c00 x1818247335694720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335694720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335694720:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019379.766248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019379.766267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019379.766274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59cad80 x1818247335694848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335694848/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766301:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019379.766308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019379.766323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019379.766329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59cbb80 x1818247335694976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766340:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335694976/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766351:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335694976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019379.766386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335695104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019379.766399:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019379.766406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59ca680 x1818247335695104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766416:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335695104/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766425:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766427:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335695104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019379.766436:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335695232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019379.766449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019379.766456:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59c9180 x1818247335695232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335695232/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335695232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019379.766485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335695360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019379.766497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019379.766503:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59c9f80 x1818247335695360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766513:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335695360/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766520:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766525:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335695360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019379.766531:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335695488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019379.766544:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019379.766550:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019379/real 1734019379] req@ffff8800b59c8700 x1818247335695488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019379.766561:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335695488/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019379.766575:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019379.766578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019379.766582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335695488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019380.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019380.727952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019381.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019381.729955:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019382.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019382.732005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019383.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019383.733928:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019384.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019384.735944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019384.773803:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.773812:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019384.773817:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019384.773827:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3649 00000100:00080000:0.0:1734019384.773835:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019384.773842:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019384.773878:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335695616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019384.773936:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.773941:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019384.773943:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019384.773948:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3649 00000100:00080000:0.0:1734019384.773952:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019384.773955:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019384.773965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019384.773977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335695744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019384.773982:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335695616:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019384.773988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.773992:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019384.773993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019384.773997:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3649 00000100:00080000:0.0:1734019384.773999:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019384.774003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019384.774007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019384.774015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335695872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019384.774024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.774027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019384.774028:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019384.774032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3649 00000100:00080000:0.0:1734019384.774034:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019384.774038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019384.774042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019384.774050:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335696000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019384.774053:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019384.774058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.774061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019384.774062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019384.774067:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3649 00000100:00080000:0.0:1734019384.774070:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019384.774073:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774075:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59c9500 x1818247335695616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734019384.774077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019384.774084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335696128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019384.774089:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335695616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019384.774093:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.774096:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019384.774097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1734019384.774097:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774099:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734019384.774101:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3649 00000100:00080000:0.0:1734019384.774103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019384.774103:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335695616:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019384.774106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019384.774110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019384.774117:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335696256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019384.774123:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335695744:192.168.201.104@tcp:38:kworker.0 00000100:00080000:0.0:1734019384.774126:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019384.774129:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019384.774131:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019384.774134:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3649 00000800:00000100:3.0:1734019384.774135:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019384.774137:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019384.774141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774141:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59cb800 x1818247335695744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:0.0:1734019384.774145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019384.774148:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335695744/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019384.774154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335696384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019384.774158:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774160:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019384.774162:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335695744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019384.774166:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335695872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019384.774176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774180:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59cb480 x1818247335695872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019384.774185:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335695872/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019384.774189:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019384.774193:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335695872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019384.774197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019384.774205:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59c8a80 x1818247335696000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019384.774213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335696000/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019384.774217:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774219:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019384.774220:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019384.774224:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335696128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019384.774231:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774234:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59c8e00 x1818247335696128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019384.774239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335696128/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019384.774243:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774245:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019384.774246:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335696128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019384.774258:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335696256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019384.774264:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774267:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59c8000 x1818247335696256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019384.774273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335696256/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019384.774277:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019384.774280:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335696256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019384.774284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335696384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019384.774291:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019384.774294:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019384/real 1734019384] req@ffff8800b59cb100 x1818247335696384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019384.774301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335696384/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019384.774305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019384.774307:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019384.774308:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335696384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019385.213974:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019385.737931:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019386.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019386.739969:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019387.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019387.741994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019388.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019388.743993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019389.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019389.746014:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019389.781858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.781871:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019389.781875:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.781886:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00080000:0.0:1734019389.781895:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.781903:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019389.781939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335696512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019389.781975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.781980:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019389.781982:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.781986:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00080000:0.0:1734019389.781989:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.781993:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019389.782002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019389.782013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335696640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019389.782023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.782026:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019389.782028:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.782031:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00080000:0.0:1734019389.782034:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.782037:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019389.782042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019389.782052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335696768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019389.782061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.782064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019389.782066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.782069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00080000:0.0:1734019389.782071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.782075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019389.782079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019389.782086:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335696896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019389.782096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.782099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019389.782100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.782104:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00100000:3.0:1734019389.782105:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335696512:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019389.782107:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.782110:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019389.782115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019389.782122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335697024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019389.782130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.782133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019389.782135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.782139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00080000:0.0:1734019389.782142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.782145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019389.782149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019389.782156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335697152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019389.782164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019389.782167:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019389.782168:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019389.782171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3654 00000100:00080000:0.0:1734019389.782174:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019389.782177:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019389.782181:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019389.782188:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335697280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019389.782202:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59ca300 x1818247335696512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335696512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782276:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335696512:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019389.782284:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696640:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019389.782303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59caa00 x1818247335696640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782330:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335696640/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696640:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019389.782351:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019389.782366:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782372:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59cb100 x1818247335696768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335696768/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782389:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782395:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019389.782427:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019389.782441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59c8000 x1818247335696896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782458:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335696896/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782466:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335696896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019389.782479:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697024:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019389.782492:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782498:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59c8e00 x1818247335697024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782508:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335697024/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782517:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782520:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019389.782526:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019389.782538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59c8a80 x1818247335697152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335697152/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782565:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782568:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019389.782573:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019389.782589:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019389.782596:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019389/real 1734019389] req@ffff8800b59cb480 x1818247335697280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019389.782605:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335697280/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019389.782613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019389.782615:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019389.782618:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019390.213867:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019390.748004:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019391.213884:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019391.749988:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019392.213871:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019392.752008:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019393.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019393.754044:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019394.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019394.755993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019394.789836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.789846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019394.789850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019394.789861:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.789870:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.789878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019394.789915:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335697408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019394.789956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.789961:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019394.789962:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019394.789967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.789970:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.789974:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019394.789983:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019394.789994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335697536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019394.790003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.790006:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019394.790008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019394.790012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.790015:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.790018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019394.790023:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019394.790031:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335697664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019394.790040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.790043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019394.790044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019394.790048:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.790050:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.790054:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019394.790058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019394.790067:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335697792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019394.790076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.790079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019394.790080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019394.790083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335697408:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019394.790084:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.790087:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.790090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019394.790094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019394.790102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335697920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019394.790110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.790113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019394.790114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019394.790118:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.790120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.790123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019394.790127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019394.790135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335698048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019394.790143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019394.790146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019394.790148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019394.790151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3659 00000100:00080000:0.0:1734019394.790154:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019394.790157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019394.790161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019394.790168:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335698176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019394.790178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59cb800 x1818247335697408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335697408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790249:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335697408:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019394.790262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335697536:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019394.790284:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59c9500 x1818247335697536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335697536/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790318:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335697536:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019394.790324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335697664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019394.790338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790344:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59c8700 x1818247335697664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335697664/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335697664:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019394.790399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335697792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019394.790413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59c9f80 x1818247335697792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335697792/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790447:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335697792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019394.790456:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697920:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019394.790469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59c9180 x1818247335697920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335697920/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790495:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790498:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335697920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019394.790504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019394.790517:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790523:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59ca680 x1818247335698048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335698048/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790541:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019394.790554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019394.790567:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019394.790573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019394/real 1734019394] req@ffff8800b59cbb80 x1818247335698176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019394.790583:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335698176/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019394.790590:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019394.790592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019394.790595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019395.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019395.757979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019396.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019396.759966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019397.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019397.761990:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019398.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019398.763973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019399.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019399.765945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019399.797829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019399.797839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019399.797844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.797855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.797864:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.797872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019399.797909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335698304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019399.797944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019399.797949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019399.797950:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.797955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.797958:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.797962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019399.797971:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019399.797982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335698432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019399.797993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019399.797996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019399.797998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.798001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.798004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.798007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019399.798012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019399.798022:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335698560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019399.798030:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019399.798033:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019399.798035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.798038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.798041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.798044:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019399.798048:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019399.798056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335698688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019399.798066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019399.798068:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335698304:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019399.798069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019399.798070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.798074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.798078:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.798081:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019399.798086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019399.798093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335698816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019399.798101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019399.798104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019399.798105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.798109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.798111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.798114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019399.798118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019399.798125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335698944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019399.798133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019399.798136:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019399.798137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019399.798140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3664 00000100:00080000:0.0:1734019399.798143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019399.798146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019399.798150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019399.798158:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335699072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019399.798165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59cad80 x1818247335698304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335698304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335698304:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019399.798250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335698432:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019399.798269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59c9c00 x1818247335698432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335698432/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798295:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798298:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798302:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335698432:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019399.798308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335698560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019399.798323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798329:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59cbb80 x1818247335698560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335698560/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335698560:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019399.798385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335698688:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019399.798402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59ca680 x1818247335698688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798420:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335698688/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798428:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335698688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019399.798439:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019399.798452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59c9180 x1818247335698816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335698816/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019399.798487:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019399.798499:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59c9f80 x1818247335698944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335698944/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335698944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019399.798541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019399.798554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019399.798560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019399/real 1734019399] req@ffff8800b59c8700 x1818247335699072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019399.798570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335699072/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019399.798577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019399.798579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019399.798583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019400.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019400.767952:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019401.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019401.769856:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019402.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019402.771837:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019403.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019403.773900:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019404.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019404.775924:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019404.805831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.805841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019404.805845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.805855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00080000:0.0:1734019404.805865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.805872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019404.805908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335699200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019404.805942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.805947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019404.805949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.805953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00080000:0.0:1734019404.805956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.805960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019404.805970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019404.805980:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335699328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019404.805990:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.805993:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019404.805995:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.805998:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00080000:0.0:1734019404.806001:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.806004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019404.806010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019404.806017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335699456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019404.806027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.806030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019404.806031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.806035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00080000:0.0:1734019404.806037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.806040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019404.806044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019404.806052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335699584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019404.806061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.806064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019404.806066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.806069:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00100000:3.0:1734019404.806069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335699200:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019404.806072:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.806075:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019404.806079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019404.806087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335699712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019404.806095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.806098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019404.806100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.806104:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00080000:0.0:1734019404.806106:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.806109:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019404.806113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019404.806121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335699840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019404.806128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019404.806131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019404.806132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019404.806135:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3669 00000100:00080000:0.0:1734019404.806138:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019404.806141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019404.806145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019404.806154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335699968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019404.806161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806192:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59c9500 x1818247335699200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806215:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335699200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335699200:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019404.806246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335699328:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019404.806265:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59cb800 x1818247335699328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335699328/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806295:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335699328:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019404.806305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335699456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019404.806319:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806325:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59cb480 x1818247335699456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806335:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335699456/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806346:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806349:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335699456:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019404.806383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335699584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019404.806397:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59c8a80 x1818247335699584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335699584/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335699584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019404.806434:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019404.806446:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806452:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59c8e00 x1818247335699712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335699712/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806472:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806475:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019404.806481:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019404.806498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59c8000 x1818247335699840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335699840/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019404.806533:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019404.806545:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019404.806551:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019404/real 1734019404] req@ffff8800b59cb100 x1818247335699968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019404.806561:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335699968/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019404.806568:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019404.806571:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019404.806612:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335699968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019405.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019405.777999:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019406.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019406.779923:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019407.213816:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019407.781991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019408.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019408.783996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019409.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019409.785972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019409.813830:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.813839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019409.813845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.813858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.813869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.813878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019409.813925:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335700096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019409.813969:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.813974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019409.813975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.813980:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.813984:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.813988:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019409.813999:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019409.814011:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335700224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019409.814020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.814023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019409.814025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.814029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.814032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.814035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019409.814040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019409.814052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335700352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019409.814060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.814063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019409.814064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.814068:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.814071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.814075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019409.814080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019409.814080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335700096:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019409.814088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335700480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019409.814098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.814101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019409.814102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.814106:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.814109:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.814112:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019409.814117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019409.814124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335700608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019409.814132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.814136:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019409.814138:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.814141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.814144:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.814148:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019409.814152:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019409.814159:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335700736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019409.814165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019409.814167:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019409.814171:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019409.814173:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019409.814177:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3674 00000100:00080000:0.0:1734019409.814180:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019409.814183:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019409.814188:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019409.814193:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59caa00 x1818247335700096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019409.814196:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335700864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019409.814214:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335700096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814226:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814229:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335700096:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019409.814241:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335700224:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019409.814259:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019409.814265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59ca300 x1818247335700224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019409.814276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335700224/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814285:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814291:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335700224:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019409.814297:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335700352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019409.814311:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019409.814317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59cb100 x1818247335700352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019409.814327:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335700352/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814343:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335700352:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019409.814370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335700480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019409.814383:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019409.814389:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59c8000 x1818247335700480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019409.814400:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335700480/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814408:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814410:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335700480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019409.814418:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335700608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019409.814429:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019409.814435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59c8e00 x1818247335700608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019409.814445:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335700608/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814455:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814459:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335700608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019409.814465:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335700736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019409.814477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019409.814483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59c8a80 x1818247335700736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019409.814533:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335700736/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814541:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814543:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814547:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335700736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019409.814553:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335700864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019409.814570:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019409.814576:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019409/real 1734019409] req@ffff8800b59cb480 x1818247335700864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019409.814587:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335700864/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019409.814595:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019409.814597:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019409.814600:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335700864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019410.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019410.787984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019411.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019411.789973:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019412.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019412.791964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019413.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019413.793977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019414.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019414.795990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019414.821840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.821849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019414.821854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.821864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.821873:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.821881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019414.821919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335700992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019414.821953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.821957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019414.821959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.821964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.821967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.821970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019414.821979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019414.821990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335701120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019414.822001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.822004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019414.822006:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.822009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.822012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.822015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019414.822019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019414.822028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335701248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019414.822036:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.822039:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019414.822041:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.822044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.822047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.822050:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019414.822054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019414.822062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335701376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019414.822071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.822074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019414.822074:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335700992:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019414.822076:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.822080:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.822083:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.822087:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019414.822091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019414.822099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335701504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019414.822107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.822110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019414.822111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.822114:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.822117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.822120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019414.822124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019414.822131:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335701632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019414.822138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019414.822141:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019414.822143:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019414.822146:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3679 00000100:00080000:0.0:1734019414.822149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019414.822153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019414.822156:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019414.822165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335701760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019414.822165:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822196:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59cb800 x1818247335700992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822218:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335700992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822231:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822240:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335700992:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019414.822248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335701120:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019414.822267:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59c9500 x1818247335701120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335701120/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335701120:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019414.822306:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335701248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019414.822322:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59c8700 x1818247335701248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335701248/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822348:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335701248:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019414.822385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335701376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019414.822398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59c9f80 x1818247335701376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335701376/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335701376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019414.822485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335701504:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019414.822502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822509:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59c9180 x1818247335701504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335701504/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335701504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019414.822541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335701632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019414.822553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59ca680 x1818247335701632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335701632/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822580:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335701632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019414.822589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335701760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019414.822602:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019414.822608:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019414/real 1734019414] req@ffff8800b59cbb80 x1818247335701760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019414.822618:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335701760/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019414.822626:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019414.822628:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019414.822632:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335701760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019415.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019415.797950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019416.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019416.799967:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019417.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019417.801994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019418.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019418.803947:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019419.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019419.805995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019419.829800:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.829810:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019419.829814:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019419.829825:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.829834:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.829842:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019419.829880:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335701888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019419.829914:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.829918:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019419.829920:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019419.829924:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.829927:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.829931:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019419.829940:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019419.829951:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335702016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019419.829960:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.829964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019419.829965:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019419.829969:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.829971:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.829975:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019419.829980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019419.829990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335702144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019419.829999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.830002:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019419.830003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019419.830007:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.830009:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.830012:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019419.830016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019419.830024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335702272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019419.830034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.830037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019419.830038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019419.830041:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335701888:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019419.830042:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.830045:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.830049:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019419.830052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019419.830061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335702400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019419.830069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.830071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019419.830073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019419.830077:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.830079:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.830083:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019419.830087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019419.830093:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335702528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019419.830101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019419.830104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019419.830106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019419.830109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3684 00000100:00080000:0.0:1734019419.830111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019419.830115:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019419.830118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019419.830126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335702656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019419.830133:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830161:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59c9c00 x1818247335701888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830184:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335701888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830196:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830201:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830206:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335701888:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019419.830214:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702016:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019419.830232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59cad80 x1818247335702016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335702016/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830264:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702016:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019419.830270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702144:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019419.830285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59cbb80 x1818247335702144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335702144/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830317:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019419.830389:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019419.830406:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830413:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59ca680 x1818247335702272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830424:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335702272/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830431:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830434:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019419.830443:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335702400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019419.830456:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830462:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59c9180 x1818247335702400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335702400/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335702400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019419.830499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335702528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019419.830511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59c9f80 x1818247335702528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335702528/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335702528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019419.830546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335702656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019419.830559:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019419.830565:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019419/real 1734019419] req@ffff8800b59c8700 x1818247335702656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019419.830575:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335702656/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019419.830582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019419.830585:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019419.830588:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335702656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019420.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019420.807956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019421.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019421.809949:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019422.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019422.811953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019423.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019423.813999:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019424.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019424.815983:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019424.837831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.837842:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019424.837846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.837857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.837866:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.837874:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019424.837909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335702784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019424.837951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.837955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019424.837957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.837962:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.837965:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.837969:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019424.837978:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019424.837989:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335702912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019424.837998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.838001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019424.838003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.838007:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.838009:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.838012:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019424.838017:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019424.838025:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335703040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019424.838033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.838036:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019424.838038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.838041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.838044:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.838048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019424.838052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019424.838060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335703168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019424.838069:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.838072:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019424.838073:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335702784:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019424.838074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.838078:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.838080:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.838083:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019424.838087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019424.838094:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335703296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019424.838102:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.838105:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019424.838106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.838110:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.838113:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.838116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019424.838120:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019424.838127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335703424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019424.838135:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019424.838138:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019424.838140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019424.838143:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3689 00000100:00080000:0.0:1734019424.838146:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019424.838150:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019424.838153:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019424.838162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335703552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019424.838198:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019424.838222:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019424.838283:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019424.838303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335703296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019424.838324:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335703424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019424.838341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335703552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019425.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019425.818003:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019426.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019426.819923:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019427.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019427.821991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1734019427.843982:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019427.844028:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019427.844048:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019427.844086:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019427.844096:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019427.844103:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019427.844112:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019427.844118:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019427.844125:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019427.844245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59c8a80 x1818247335703168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844286:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335703168/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844323:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019427.844337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59c8e00 x1818247335703296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335703296/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335703296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019427.844395:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59c8000 x1818247335703424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335703424/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335703424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019427.844449:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59cb100 x1818247335703552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335703552/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335703552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019427.844518:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59c9500 x1818247335702784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335702784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844552:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335702784:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019427.844573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59cb800 x1818247335702912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335702912/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844606:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844611:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844616:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335702912:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019427.844628:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019424/real 1734019427] req@ffff8800b59cb480 x1818247335703040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019427.844647:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335703040/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019427.844659:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019427.844663:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019427.844669:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703040:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019428.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019428.823982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019429.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019429.825992:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019429.845800:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.845810:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019429.845815:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.845830:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.845841:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.845851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019429.845897:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335703680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019429.845933:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.845937:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019429.845940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.845945:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.845948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.845952:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019429.845963:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019429.845974:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335703808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019429.845986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.845990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019429.845991:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.845995:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.845997:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.846001:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019429.846006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019429.846016:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335703936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019429.846024:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.846027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019429.846029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.846032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.846035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.846039:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846042:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335703680:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019429.846043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019429.846052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335704064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019429.846061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.846064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019429.846066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.846069:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.846073:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.846076:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019429.846080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019429.846088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335704192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019429.846096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.846099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019429.846101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.846104:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.846107:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.846111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019429.846115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019429.846123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335704320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019429.846128:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019429.846131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019429.846134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019429.846135:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019429.846139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3694 00000100:00080000:0.0:1734019429.846142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019429.846145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019429.846149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019429.846195:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335704448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019429.846198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59ca300 x1818247335703680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846221:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335703680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335703680:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019429.846250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703808:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019429.846271:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846279:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59caa00 x1818247335703808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846290:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335703808/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846299:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846301:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846304:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703808:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019429.846310:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019429.846325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846331:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59cb480 x1818247335703936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846343:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335703936/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846353:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335703936:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019429.846385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019429.846398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846405:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59cb800 x1818247335704064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335704064/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846422:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846425:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846428:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019429.846433:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335704192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019429.846445:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59c9500 x1818247335704192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846461:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335704192/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846468:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846471:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846474:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335704192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019429.846480:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335704320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019429.846492:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846498:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59cb100 x1818247335704320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846507:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335704320/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846515:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846517:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846521:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335704320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019429.846526:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335704448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019429.846539:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019429.846544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019429/real 1734019429] req@ffff8800b59c8000 x1818247335704448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019429.846554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335704448/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019429.846562:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019429.846564:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019429.846567:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335704448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019430.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019430.827966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019431.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019431.829970:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019432.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019432.831999:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019433.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019433.833992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019434.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019434.835950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019434.853834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.853845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019434.853850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.853860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.853869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.853877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019434.853914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335704576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019434.853949:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.853953:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019434.853955:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.853959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.853962:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.853966:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019434.853975:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019434.853986:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335704704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019434.853995:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.853998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019434.854000:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.854004:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.854006:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.854010:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019434.854015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019434.854023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335704832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019434.854032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.854035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019434.854037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.854040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.854042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.854046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019434.854050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019434.854057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335704960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019434.854095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335704576:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019434.854099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.854104:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019434.854106:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.854110:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.854113:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.854117:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019434.854121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019434.854130:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335705088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019434.854139:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.854142:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019434.854144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.854148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.854151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.854154:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019434.854158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019434.854166:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335705216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019434.854173:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019434.854176:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019434.854178:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019434.854181:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3699 00000100:00080000:0.0:1734019434.854184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019434.854187:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019434.854191:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019434.854191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019434.854200:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335705344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019434.854222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59c8e00 x1818247335704576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335704576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854262:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335704576:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019434.854276:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704704:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019434.854297:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019434.854303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59c8a80 x1818247335704704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335704704/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854327:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854330:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704704:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019434.854336:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019434.854351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019434.854359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59c8700 x1818247335704832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335704832/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854378:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854380:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854383:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704832:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019434.854417:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019434.854430:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019434.854436:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59c9f80 x1818247335704960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854446:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335704960/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854453:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854456:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854459:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335704960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019434.854466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019434.854478:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019434.854484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59c9180 x1818247335705088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854494:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335705088/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854510:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854512:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854515:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019434.854521:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019434.854533:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019434.854539:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59ca680 x1818247335705216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854549:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335705216/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019434.854567:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019434.854580:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019434.854586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019434/real 1734019434] req@ffff8800b59cbb80 x1818247335705344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019434.854595:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335705344/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019434.854603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019434.854605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019434.854608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019435.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019435.837983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019436.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019436.839970:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019437.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019437.842002:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019438.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019438.843940:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019439.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019439.846012:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019439.861834:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.861844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019439.861848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019439.861859:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.861868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.861876:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019439.861911:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335705472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019439.861953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.861958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019439.861959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019439.861964:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.861967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.861971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019439.861980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019439.862040:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335705600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019439.862051:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.862054:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019439.862056:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019439.862059:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.862062:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.862066:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019439.862071:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019439.862081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335705728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019439.862092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.862095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019439.862097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019439.862097:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335705472:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019439.862100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.862103:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.862107:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019439.862111:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019439.862120:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335705856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019439.862128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.862131:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019439.862133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019439.862137:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.862139:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.862143:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019439.862146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019439.862153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335705984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019439.862161:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.862164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019439.862165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019439.862169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.862172:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.862175:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019439.862179:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019439.862187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335706112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019439.862191:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019439.862194:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019439.862198:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019439.862199:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019439.862203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3704 00000100:00080000:0.0:1734019439.862206:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019439.862209:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019439.862213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019439.862221:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335706240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019439.862222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59cad80 x1818247335705472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335705472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862257:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335705472:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019439.862275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335705600:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019439.862295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019439.862302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59c9c00 x1818247335705600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862313:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335705600/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335705600:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019439.862363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335705728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019439.862378:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019439.862385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59cbb80 x1818247335705728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335705728/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862407:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335705728:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019439.862415:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335705856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019439.862428:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019439.862434:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59ca680 x1818247335705856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335705856/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335705856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019439.862463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019439.862477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019439.862484:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59c9180 x1818247335705984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335705984/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335705984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019439.862515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335706112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019439.862527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019439.862533:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59c9f80 x1818247335706112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862543:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335706112/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862553:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862556:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335706112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019439.862561:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335706240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019439.862575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019439.862581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019439/real 1734019439] req@ffff8800b59c8700 x1818247335706240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019439.862592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335706240/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019439.862599:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019439.862603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019439.862606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335706240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019440.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019440.848013:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019441.213800:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019441.849989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019442.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019442.851885:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019443.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019443.854006:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019444.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019444.855989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019444.869837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.869847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019444.869852:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.869866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.869876:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.869886:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019444.869971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335706368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019444.870006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.870011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019444.870013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.870018:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.870022:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.870025:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019444.870037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019444.870049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335706496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019444.870060:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.870063:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019444.870065:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.870069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.870071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.870075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019444.870079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019444.870088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335706624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019444.870096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.870099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019444.870101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.870104:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.870107:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.870110:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019444.870114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019444.870116:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335706368:192.168.201.104@tcp:250:kworker.0 00000100:00100000:0.0:1734019444.870122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335706752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019444.870131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.870135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019444.870136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.870140:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.870144:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.870147:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019444.870151:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019444.870159:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335706880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019444.870167:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.870170:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019444.870171:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.870175:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.870177:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.870181:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019444.870185:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019444.870192:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335707008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019444.870199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019444.870200:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019444.870203:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019444.870205:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019444.870208:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3709 00000100:00080000:0.0:1734019444.870211:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019444.870214:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019444.870218:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019444.870226:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59c8a80 x1818247335706368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019444.870228:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335707136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019444.870248:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335706368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870268:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335706368:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019444.870275:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335706496:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019444.870293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019444.870300:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59c8e00 x1818247335706496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019444.870310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335706496/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870319:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870326:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335706496:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019444.870332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335706624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019444.870348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019444.870354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59c8000 x1818247335706624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019444.870364:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335706624/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870372:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870375:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870378:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335706624:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019444.870407:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335706752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019444.870419:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019444.870425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59cb100 x1818247335706752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019444.870436:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335706752/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870444:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870449:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335706752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019444.870454:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335706880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019444.870466:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019444.870473:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59c9500 x1818247335706880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019444.870484:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335706880/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870491:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870493:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870496:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335706880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019444.870501:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019444.870513:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019444.870519:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59cb800 x1818247335707008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019444.870528:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335707008/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870536:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870538:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870541:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019444.870546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019444.870558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019444.870564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019444/real 1734019444] req@ffff8800b59cb480 x1818247335707136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019444.870574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335707136/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019444.870589:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019444.870592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019444.870595:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019445.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019445.857979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019446.213763:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019446.859945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019447.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019447.861994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019448.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019448.863932:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019449.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019449.865961:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019449.877804:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019449.877813:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019449.877840:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019449.877850:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.877859:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.877867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019449.877901:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335707264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019449.877934:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019449.877938:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019449.877940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019449.877945:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.877948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.877951:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019449.877961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019449.877971:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335707392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019449.877982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019449.877984:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335707264:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019449.877985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019449.877987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019449.877991:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.877993:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.877997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019449.878002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019449.878012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335707520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019449.878021:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019449.878024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019449.878025:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019449.878029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.878032:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.878035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019449.878039:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019449.878046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335707648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019449.878054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019449.878057:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019449.878058:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1734019449.878060:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019449.878062:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.878065:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.878068:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019449.878072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019449.878079:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335707776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019449.878087:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019449.878090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1734019449.878090:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59caa00 x1818247335707264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:0.0:1734019449.878092:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019449.878095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.878098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.878101:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019449.878105:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019449.878109:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335707264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019449.878112:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335707904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019449.878120:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1734019449.878121:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019449.878124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1734019449.878124:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:0.0:1734019449.878125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019449.878128:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335707264:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019449.878129:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3714 00000100:00080000:0.0:1734019449.878131:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019449.878134:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019449.878138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019449.878146:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335708032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019449.878156:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335707392:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019449.878176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019449.878184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59ca300 x1818247335707392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019449.878195:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335707392/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019449.878210:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019449.878213:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019449.878216:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335707392:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019449.878222:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335707520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019449.878237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019449.878244:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59cb480 x1818247335707520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019449.878254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335707520/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019449.878262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019449.878264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019449.878267:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335707520:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019449.878274:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335707648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019449.878287:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019449.878293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59cb800 x1818247335707648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019449.878303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335707648/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019449.878310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019449.878312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019449.878314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335707648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019449.878321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019449.878334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019449.878339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59c9500 x1818247335707776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019449.878349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335707776/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019449.878356:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019449.878358:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019449.878361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019449.878366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019449.878377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019449.878382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59cb100 x1818247335707904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019449.878391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335707904/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019449.878398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019449.878400:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019449.878403:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335707904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019449.878421:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019449.878434:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019449.878440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019449/real 1734019449] req@ffff8800b59c8000 x1818247335708032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019449.878452:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335708032/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019449.878459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019449.878461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019449.878464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019450.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019450.867951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019451.213893:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019451.870065:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019452.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019452.872007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019453.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019453.874024:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019454.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019454.875981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019454.885840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.885851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019454.885857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.885870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.885882:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.885891:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019454.885938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335708160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019454.885982:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.885987:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019454.885988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.885994:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.885997:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.886001:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019454.886012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019454.886024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335708288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019454.886034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.886038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019454.886039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.886043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.886046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.886049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019454.886054:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019454.886062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335708416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019454.886070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.886073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019454.886075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.886079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.886081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.886086:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886088:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335708160:192.168.201.104@tcp:250:kworker.0 00000008:00100000:0.0:1734019454.886090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019454.886099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335708544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019454.886107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.886111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019454.886112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.886116:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.886118:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.886122:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019454.886126:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019454.886133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335708672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019454.886141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.886144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019454.886146:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.886149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.886152:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.886156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019454.886161:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019454.886169:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335708800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019454.886170:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:0.0:1734019454.886177:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019454.886181:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019454.886183:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019454.886187:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3719 00000100:00080000:0.0:1734019454.886190:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019454.886193:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019454.886197:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019454.886198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59c8e00 x1818247335708160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:0.0:1734019454.886205:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335708928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019454.886220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335708160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886241:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335708160:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019454.886248:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335708288:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019454.886266:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59c8a80 x1818247335708288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019454.886292:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335708288/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335708288:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019454.886312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335708416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019454.886326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59c8700 x1818247335708416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019454.886342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335708416/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886355:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335708416:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019454.886386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335708544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019454.886400:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886406:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59c9f80 x1818247335708544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019454.886417:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335708544/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886425:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335708544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019454.886437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019454.886449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59c9180 x1818247335708672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019454.886465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335708672/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886472:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886478:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019454.886483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019454.886494:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59ca680 x1818247335708800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019454.886510:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335708800/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886517:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886519:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886522:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019454.886527:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019454.886543:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019454.886549:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019454/real 1734019454] req@ffff8800b59cbb80 x1818247335708928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019454.886559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335708928/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019454.886566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019454.886568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019454.886571:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335708928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019455.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019455.877963:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019456.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019456.879983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019457.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019457.881971:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019458.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019458.883949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019459.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019459.885985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019459.893838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.893850:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019459.893854:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019459.893866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.893875:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.893883:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019459.893921:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9c00 x1818247335709056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019459.893956:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.893960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019459.893962:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019459.893966:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.893969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.893973:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019459.893982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019459.893993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cad80 x1818247335709184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019459.894004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.894007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019459.894009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019459.894012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.894014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.894018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019459.894022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019459.894032:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335709312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019459.894040:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.894043:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019459.894045:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019459.894048:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.894051:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.894054:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019459.894058:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019459.894065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335709440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019459.894075:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.894078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019459.894080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019459.894081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335709056:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019459.894083:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.894086:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.894090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019459.894094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019459.894102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335709568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019459.894110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.894113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019459.894114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019459.894117:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.894120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.894123:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019459.894127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019459.894133:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335709696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019459.894141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019459.894144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019459.894146:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019459.894149:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3724 00000100:00080000:0.0:1734019459.894151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019459.894154:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019459.894158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019459.894165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335709824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019459.894181:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894211:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59c9c00 x1818247335709056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894233:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9c00 x1818247335709056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894255:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335709056:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019459.894263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335709184:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019459.894285:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59cad80 x1818247335709184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894305:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cad80 x1818247335709184/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894313:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894316:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335709184:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019459.894325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335709312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019459.894340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894346:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59cbb80 x1818247335709312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335709312/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894367:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335709312:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019459.894404:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335709440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019459.894418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894425:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59ca680 x1818247335709440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894443:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335709440/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335709440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019459.894463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335709568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019459.894475:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894481:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59c9180 x1818247335709568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894491:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335709568/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894498:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894501:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894504:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335709568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019459.894510:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335709696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019459.894523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59c9f80 x1818247335709696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335709696/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335709696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019459.894560:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335709824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019459.894573:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019459.894579:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019459/real 1734019459] req@ffff8800b59c8700 x1818247335709824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019459.894588:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335709824/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019459.894596:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019459.894598:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019459.894601:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335709824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019460.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019460.887992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019461.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019461.889966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019462.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019462.891948:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019463.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019463.893978:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019464.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019464.895946:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019464.901850:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.901860:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019464.901865:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019464.901875:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.901885:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.901892:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019464.901929:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335709952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019464.901966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.901970:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019464.901972:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019464.901976:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.901979:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.901983:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019464.901992:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019464.902003:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335710080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019464.902012:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.902015:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019464.902017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019464.902021:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.902024:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.902027:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019464.902032:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019464.902041:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335710208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019464.902050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.902053:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019464.902054:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019464.902058:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.902060:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.902064:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019464.902068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019464.902075:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335710336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019464.902085:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.902088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019464.902089:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019464.902091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335709952:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019464.902093:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.902096:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.902099:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019464.902103:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019464.902111:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335710464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019464.902119:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.902122:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019464.902124:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019464.902127:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.902130:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.902133:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019464.902137:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019464.902144:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335710592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019464.902151:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019464.902154:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019464.902156:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019464.902159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3729 00000100:00080000:0.0:1734019464.902162:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019464.902165:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019464.902169:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019464.902178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335710720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019464.902184:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902215:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59c8a80 x1818247335709952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902237:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335709952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335709952:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019464.902270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710080:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019464.902289:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59c8e00 x1818247335710080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335710080/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902315:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710080:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019464.902328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019464.902343:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902349:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59c8000 x1818247335710208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335710208/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902366:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902373:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710208:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019464.902402:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710336:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019464.902420:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902426:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59cb100 x1818247335710336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902437:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335710336/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902451:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019464.902457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335710464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019464.902469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59c9500 x1818247335710464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335710464/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335710464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019464.902505:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335710592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019464.902516:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902522:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59cb800 x1818247335710592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335710592/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902551:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902554:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335710592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019464.902559:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335710720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019464.902605:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019464.902612:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019464/real 1734019464] req@ffff8800b59cb480 x1818247335710720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019464.902623:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335710720/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019464.902630:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019464.902632:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019464.902636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335710720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019465.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019465.897991:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019466.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019466.899968:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019467.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019467.901977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019468.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019468.903985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019469.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019469.905997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019469.909836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.909846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019469.909850:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.909860:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.909869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019469.909877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019469.909913:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca300 x1818247335710848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019469.909954:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.909959:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019469.909960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.909965:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.909968:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019469.909972:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019469.909981:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019469.909992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59caa00 x1818247335710976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019469.910000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.910004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019469.910005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.910009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.910011:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019469.910015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019469.910019:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019469.910030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb480 x1818247335711104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019469.910038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.910041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019469.910043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.910046:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.910049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019469.910053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019469.910057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019469.910065:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb800 x1818247335711232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019469.910074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.910077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019469.910078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.910082:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.910085:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019469.910085:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335710848:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019469.910088:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019469.910092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019469.910100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9500 x1818247335711360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019469.910107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.910110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019469.910112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.910115:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.910117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019469.910120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019469.910125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019469.910132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cb100 x1818247335711488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019469.910140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019469.910143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019469.910145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019469.910148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3734 00000100:00080000:0.0:1734019469.910151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019469.910154:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019469.910158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019469.910165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8000 x1818247335711616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019469.910178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910209:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59ca300 x1818247335710848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca300 x1818247335710848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910249:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910254:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335710848:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019469.910262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710976:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019469.910281:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59caa00 x1818247335710976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59caa00 x1818247335710976/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910311:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335710976:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019469.910320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335711104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019469.910335:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59cb480 x1818247335711104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb480 x1818247335711104/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335711104:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019469.910399:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335711232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019469.910413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59cb800 x1818247335711232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910430:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb800 x1818247335711232/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910443:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335711232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019469.910449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335711360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019469.910461:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59c9500 x1818247335711360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910523:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9500 x1818247335711360/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910535:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335711360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019469.910545:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335711488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019469.910565:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910573:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59cb100 x1818247335711488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910585:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cb100 x1818247335711488/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910593:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910595:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335711488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019469.910604:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335711616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019469.910617:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019469.910623:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019469/real 1734019469] req@ffff8800b59c8000 x1818247335711616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019469.910633:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8000 x1818247335711616/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019469.910641:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019469.910643:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019469.910646:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335711616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019470.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019470.907993:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019471.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019471.909985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019472.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019472.911967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019473.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019473.913971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019474.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019474.915972:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1734019474.917793:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.917801:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1734019474.917806:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.917816:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.917825:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.917833:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019474.917868:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8e00 x1818247335711744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019474.917902:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.917906:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019474.917908:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.917913:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.917916:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.917919:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019474.917929:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019474.917940:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8a80 x1818247335711872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019474.917951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.917954:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019474.917956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.917959:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.917962:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.917965:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019474.917969:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:0.0:1734019474.917978:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c8700 x1818247335712000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019474.917986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.917989:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019474.917990:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.917994:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.917996:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.918000:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019474.918004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019474.918012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9f80 x1818247335712128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019474.918020:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335711744:192.168.201.104@tcp:250:kworker.0 00000100:00080000:0.0:1734019474.918022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.918025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1734019474.918026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.918029:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.918032:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.918036:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019474.918040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1734019474.918047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59c9180 x1818247335712256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019474.918055:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.918058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1734019474.918059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.918063:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.918065:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.918068:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019474.918072:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1734019474.918079:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59ca680 x1818247335712384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:0.0:1734019474.918087:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1734019474.918090:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1734019474.918091:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1734019474.918095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3739 00000100:00080000:0.0:1734019474.918098:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1734019474.918101:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1734019474.918105:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019474.918113:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:0.0:1734019474.918114:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800b59cbb80 x1818247335712512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019474.918142:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59c8e00 x1818247335711744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918165:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8e00 x1818247335711744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918179:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918183:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918188:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335711744:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019474.918196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335711872:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019474.918215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019474.918221:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59c8a80 x1818247335711872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8a80 x1818247335711872/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918244:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335711872:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019474.918254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019474.918269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019474.918275:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59c8700 x1818247335712000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c8700 x1818247335712000/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918297:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918300:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712000:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019474.918331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019474.918344:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019474.918351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59c9f80 x1818247335712128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918361:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9f80 x1818247335712128/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918372:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918375:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019474.918381:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335712256:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019474.918436:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019474.918445:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59c9180 x1818247335712256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918456:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59c9180 x1818247335712256/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918463:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918466:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918469:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335712256:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019474.918477:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335712384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019474.918493:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019474.918500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59ca680 x1818247335712384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59ca680 x1818247335712384/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918519:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335712384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019474.918529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335712512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019474.918543:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019474.918549:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019474/real 1734019474] req@ffff8800b59cbb80 x1818247335712512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019474.918559:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800b59cbb80 x1818247335712512/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019474.918566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019474.918568:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019474.918572:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800b59cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335712512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019475.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019475.917953:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019476.213952:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019476.919991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019477.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019477.921979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019478.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019478.923960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019479.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019479.925960:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.925972:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019479.925978:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019479.925991:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926002:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019479.926012:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00010000:00010000:3.0:1734019479.926012:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019479.926060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335712640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019479.926101:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.926106:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019479.926108:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019479.926113:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926116:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019479.926120:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019479.926131:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019479.926139:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335712640:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019479.926143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335712768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019479.926154:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.926158:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019479.926159:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019479.926163:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926166:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019479.926169:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019479.926174:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019479.926182:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335712896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019479.926191:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.926194:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019479.926196:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019479.926200:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926213:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019479.926216:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019479.926220:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019479.926226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019479.926233:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335713024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019479.926241:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.926244:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019479.926246:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019479.926250:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926253:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019479.926254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86dca80 x1818247335712640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019479.926257:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019479.926262:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019479.926271:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335713152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019479.926276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335712640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019479.926279:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.926282:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019479.926284:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1734019479.926287:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734019479.926288:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926290:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1734019479.926290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734019479.926294:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926296:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335712640:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019479.926298:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019479.926306:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335713280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019479.926348:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019479.926351:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019479.926352:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019479.926356:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3744 00000100:00080000:2.0:1734019479.926359:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019479.926361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712768:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734019479.926363:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019479.926368:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019479.926377:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335713408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019479.926383:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86dd880 x1818247335712768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019479.926404:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335712768/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019479.926413:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019479.926415:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019479.926419:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712768:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019479.926425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712896:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019479.926440:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86df480 x1818247335712896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019479.926458:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335712896/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019479.926467:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019479.926469:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019479.926472:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335712896:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019479.926478:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019479.926490:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86de680 x1818247335713024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019479.926506:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335713024/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019479.926513:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019479.926515:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019479.926518:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019479.926524:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335713152:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019479.926538:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926544:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86dc700 x1818247335713152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019479.926555:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335713152/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019479.926563:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019479.926566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019479.926569:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335713152:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019479.926574:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335713280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019479.926587:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926593:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86df100 x1818247335713280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019479.926603:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335713280/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019479.926610:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019479.926613:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019479.926616:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335713280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019479.926634:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335713408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019479.926647:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019479.926653:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019479/real 1734019479] req@ffff8800a86dd500 x1818247335713408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019479.926664:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335713408/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019479.926672:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019479.926674:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019479.926677:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335713408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:0.0:1734019480.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019480.927965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019481.213936:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019481.929963:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019482.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019482.931959:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019483.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019483.934023:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019484.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019484.933842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.933853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019484.933858:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.933867:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.933877:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.933884:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019484.933918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335713536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019484.933953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.933957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019484.933959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.933963:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.933967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.933970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019484.933980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019484.933990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335713664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019484.934001:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.934004:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019484.934005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.934009:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.934012:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.934015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019484.934020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019484.934028:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335713792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019484.934039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.934042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019484.934044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.934048:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.934050:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.934053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019484.934057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019484.934068:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335713920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019484.934069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335713536:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019484.934076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.934079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019484.934081:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.934084:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.934087:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.934090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019484.934094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019484.934102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335714048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019484.934110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.934113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019484.934115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.934119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.934121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.934125:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019484.934128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019484.934135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335714176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019484.934143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019484.934146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019484.934148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019484.934151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3749 00000100:00080000:2.0:1734019484.934163:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019484.934166:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019484.934170:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019484.934174:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019484.934178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335714304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019484.934206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86dd180 x1818247335713536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934275:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335713536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934294:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934299:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335713536:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019484.934307:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713664:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019484.934333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019484.934341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86dc000 x1818247335713664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934353:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335713664/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934362:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713664:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019484.934396:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713792:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019484.934411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019484.934417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86ddc00 x1818247335713792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335713792/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934438:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713792:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019484.934450:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019484.934462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019484.934468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86ddf80 x1818247335713920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934478:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335713920/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934486:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335713920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019484.934497:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714048:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019484.934510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019484.934517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86dfb80 x1818247335714048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335714048/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714048:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019484.934547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019484.934560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019484.934566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86df800 x1818247335714176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335714176/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934587:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934590:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019484.934596:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019484.934609:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019484.934615:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019484/real 1734019484] req@ffff8800a86de300 x1818247335714304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019484.934625:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335714304/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019484.934632:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019484.934635:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019484.934638:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714304:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019484.935930:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019485.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019485.937968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019486.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019486.939956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019487.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019487.941980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019488.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019488.943909:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019489.213753:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019489.941771:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019489.941779:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019489.941782:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941788:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941795:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941800:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019489.941824:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335714432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019489.941845:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019489.941847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019489.941848:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941851:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941853:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941854:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019489.941860:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019489.941866:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335714560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019489.941871:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019489.941873:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019489.941874:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941876:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941877:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941879:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019489.941881:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019489.941885:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335714688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019489.941889:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019489.941892:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019489.941892:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941895:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941896:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941898:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019489.941900:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019489.941906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335714816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019489.941910:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019489.941912:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019489.941912:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941914:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941916:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941917:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019489.941920:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019489.941924:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335714944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019489.941928:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019489.941929:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019489.941930:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941932:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941933:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941935:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019489.941937:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019489.941942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335715072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019489.941947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1734019489.941947:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335714432:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019489.941949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019489.941949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019489.941951:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3754 00000100:00080000:2.0:1734019489.941952:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019489.941954:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019489.941956:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019489.941960:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335715200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019489.942034:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335714560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019489.942048:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335714688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019489.942059:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335714816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019489.942069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714944:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019489.942086:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019489.942094:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715200:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019489.945795:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019490.213748:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019490.947948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019491.213756:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019491.949945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019492.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734019492.948045:0:1956:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019492.948130:0:1956:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019492.948155:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019492.948201:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019492.948210:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019492.948218:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019492.948225:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019492.948233:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019492.948241:0:1956:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019492.948308:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86dfb80 x1818247335715072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335715072/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948355:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948360:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948366:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019492.948375:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86ddf80 x1818247335715200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948404:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335715200/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948418:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948423:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948429:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019492.948440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86dce00 x1818247335714432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948459:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335714432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948471:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948475:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948481:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335714432:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019492.948492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86dea00 x1818247335714560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335714560/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948528:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948534:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335714560:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019492.948545:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86dc380 x1818247335714688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335714688/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948592:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948598:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335714688:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019492.948619:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86de300 x1818247335714816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948639:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335714816/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948651:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948654:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948660:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335714816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019492.948670:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019489/real 1734019492] req@ffff8800a86df800 x1818247335714944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019492.948688:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335714944/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019492.948976:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019492.948980:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019492.948986:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335714944:192.168.201.104@tcp:38:kworker.0 00010000:00010000:3.0:1734019492.951880:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019493.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019493.953859:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019494.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019494.949830:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.949840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019494.949844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.949855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.949864:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.949872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019494.949908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335715328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019494.949951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.949955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019494.949957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.949961:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.949965:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.949968:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019494.949978:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019494.949988:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335715456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019494.949998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.950001:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019494.950002:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.950006:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.950008:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.950011:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019494.950016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019494.950023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335715584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019494.950032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.950035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019494.950036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.950040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.950043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.950047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019494.950051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019494.950099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335715712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019494.950109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.950113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019494.950115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.950118:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.950121:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.950124:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950127:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335715328:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019494.950129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019494.950138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335715840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019494.950146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.950149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019494.950151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.950154:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.950158:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.950161:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019494.950165:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019494.950172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335715968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019494.950180:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019494.950183:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019494.950185:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019494.950188:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3759 00000100:00080000:2.0:1734019494.950208:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019494.950211:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019494.950215:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019494.950222:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019494.950224:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335716096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019494.950253:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86ddc00 x1818247335715328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950275:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335715328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950288:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950292:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950298:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335715328:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019494.950305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335715456:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019494.950325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86dc000 x1818247335715456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335715456/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335715456:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019494.950364:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335715584:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019494.950379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950385:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86dd180 x1818247335715584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950403:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335715584/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950412:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335715584:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019494.950447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335715712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019494.950460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950466:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86dd500 x1818247335715712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950476:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335715712/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950484:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950490:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335715712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019494.950496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715840:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019494.950509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950515:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86df100 x1818247335715840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335715840/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715840:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019494.950546:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019494.950558:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950564:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86dc700 x1818247335715968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950574:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335715968/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950582:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950584:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950587:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335715968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019494.950593:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019494.950606:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019494.950612:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019494/real 1734019494] req@ffff8800a86de680 x1818247335716096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019494.950623:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335716096/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019494.950630:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019494.950632:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019494.950636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716096:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019494.955919:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019495.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019495.957956:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019496.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019496.959996:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019497.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019497.961982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019498.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019498.963976:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019499.213850:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019499.957831:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.957841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019499.957846:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019499.957857:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.957865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.957873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019499.957908:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335716224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019499.957944:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.957949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019499.957951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019499.957955:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.957959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.957963:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019499.958009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019499.958024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335716352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019499.958035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.958038:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019499.958040:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019499.958044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.958047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.958050:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019499.958055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019499.958063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335716480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019499.958073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.958076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019499.958078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019499.958078:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335716224:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019499.958082:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.958084:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.958088:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019499.958092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019499.958103:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335716608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019499.958117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.958121:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019499.958123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019499.958126:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.958129:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.958132:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019499.958136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019499.958143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335716736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019499.958151:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.958154:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019499.958155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019499.958159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.958162:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.958165:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019499.958169:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019499.958171:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019499.958176:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335716864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019499.958184:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019499.958187:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019499.958188:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019499.958192:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3764 00000100:00080000:2.0:1734019499.958195:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019499.958198:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019499.958202:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019499.958203:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86df480 x1818247335716224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734019499.958210:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335716992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019499.958225:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335716224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958238:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958243:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958248:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335716224:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019499.958257:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335716352:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019499.958276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019499.958283:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86dd880 x1818247335716352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019499.958294:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335716352/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958303:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335716352:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019499.958337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335716480:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019499.958352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019499.958359:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86dca80 x1818247335716480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019499.958370:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335716480/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958381:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958384:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335716480:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019499.958390:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335716608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019499.958403:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019499.958409:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86de680 x1818247335716608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019499.958418:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335716608/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958426:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958428:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958431:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335716608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019499.958437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716736:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019499.958451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019499.958457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86dc700 x1818247335716736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019499.958467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335716736/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958482:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958485:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958488:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716736:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019499.958494:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019499.958507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019499.958513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86df100 x1818247335716864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019499.958522:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335716864/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958530:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019499.958541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019499.958553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019499.958559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019499/real 1734019499] req@ffff8800a86dd500 x1818247335716992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019499.958569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335716992/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019499.958576:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019499.958578:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019499.958581:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335716992:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019499.965938:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019500.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019500.968045:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019501.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019501.970011:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019502.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019502.972009:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019503.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019503.974019:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019504.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019504.965802:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.965812:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019504.965817:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.965827:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.965837:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.965844:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019504.965879:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335717120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019504.965941:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.965946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019504.965948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.965953:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.965956:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.965960:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019504.965970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019504.965982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335717248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019504.965992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.965995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019504.965997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.966000:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.966002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.966006:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019504.966010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019504.966018:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335717376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019504.966027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.966030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019504.966031:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.966034:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.966037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.966040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019504.966044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019504.966053:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335717504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019504.966063:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.966066:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019504.966067:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.966071:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.966074:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.966077:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019504.966082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019504.966086:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335717120:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019504.966090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335717632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019504.966098:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.966101:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019504.966102:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.966105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.966108:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.966111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019504.966115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019504.966122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335717760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019504.966129:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019504.966132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019504.966134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019504.966137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3769 00000100:00080000:2.0:1734019504.966139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019504.966142:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019504.966146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019504.966154:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335717888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019504.966180:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966210:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86dd180 x1818247335717120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966232:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335717120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966245:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966249:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966255:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335717120:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019504.966263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335717248:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019504.966283:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966290:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86dc000 x1818247335717248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335717248/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966310:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966313:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966316:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335717248:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019504.966326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335717376:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019504.966341:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86ddc00 x1818247335717376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966359:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335717376/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966367:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966369:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335717376:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019504.966400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335717504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019504.966413:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966419:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86df800 x1818247335717504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966429:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335717504/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966437:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966440:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966444:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335717504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019504.966449:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335717632:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019504.966462:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966468:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86de300 x1818247335717632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335717632/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966487:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966493:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335717632:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019504.966499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335717760:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019504.966510:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86dc380 x1818247335717760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335717760/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335717760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019504.966544:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335717888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019504.966557:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019504.966563:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019504/real 1734019504] req@ffff8800a86dea00 x1818247335717888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019504.966572:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335717888/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019504.966580:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019504.966583:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019504.966586:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335717888:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019504.975966:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019505.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019505.977977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019506.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019506.979971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019507.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019507.982044:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019508.213870:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019508.983981:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019509.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019509.973863:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.973873:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019509.973878:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.973888:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.973897:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.973906:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019509.973942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335718016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019509.973974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.973979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019509.973981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.973985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.973988:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.973991:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019509.974001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019509.974012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335718144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019509.974021:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.974024:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019509.974026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.974030:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.974033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.974036:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019509.974040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019509.974048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335718272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019509.974057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.974060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019509.974062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.974065:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.974068:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.974071:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019509.974075:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019509.974084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335718400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019509.974092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.974095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019509.974097:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.974100:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.974102:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.974106:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019509.974110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019509.974118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335718528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019509.974122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335718016:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019509.974128:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.974132:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019509.974133:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.974137:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.974139:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.974142:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019509.974146:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019509.974153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335718656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019509.974161:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019509.974164:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019509.974165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019509.974169:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3774 00000100:00080000:2.0:1734019509.974171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019509.974174:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019509.974178:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019509.974185:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335718784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019509.974214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974245:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86dce00 x1818247335718016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974268:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335718016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974290:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335718016:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019509.974298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335718144:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019509.974317:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974324:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86ddf80 x1818247335718144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974336:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335718144/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974344:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974347:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335718144:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019509.974358:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335718272:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019509.974373:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86dfb80 x1818247335718272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974390:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335718272/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974404:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335718272:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019509.974409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335718400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019509.974422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86dea00 x1818247335718400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335718400/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335718400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019509.974484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335718528:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019509.974497:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86dc380 x1818247335718528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974516:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335718528/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974524:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335718528:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019509.974534:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335718656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019509.974546:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974552:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86de300 x1818247335718656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974562:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335718656/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974569:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974572:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974575:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335718656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019509.974580:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335718784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019509.974592:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019509.974598:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019509/real 1734019509] req@ffff8800a86df800 x1818247335718784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019509.974608:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335718784/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019509.974623:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019509.974625:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019509.974628:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335718784:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019509.985966:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019510.213881:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019510.988013:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019511.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019511.990016:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019512.213864:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019512.991995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019513.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019513.993990:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019514.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019514.981827:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.981837:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019514.981841:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.981851:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.981860:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019514.981867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019514.981906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335718912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019514.981942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.981946:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019514.981948:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.981952:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.981955:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019514.981959:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019514.981968:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019514.981979:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335719040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019514.981988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.981991:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019514.981993:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.981997:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.982000:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019514.982003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019514.982007:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019514.982015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335719168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019514.982025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.982028:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019514.982029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.982033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.982035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019514.982038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019514.982042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019514.982052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335719296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019514.982062:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.982065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019514.982066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.982070:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.982072:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019514.982072:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335718912:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019514.982076:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019514.982080:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019514.982088:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335719424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019514.982096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.982099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019514.982101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.982105:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.982107:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019514.982111:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019514.982115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019514.982122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335719552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019514.982130:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019514.982133:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019514.982134:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019514.982138:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3779 00000100:00080000:2.0:1734019514.982156:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019514.982159:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019514.982163:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019514.982167:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019514.982171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335719680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019514.982197:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86ddc00 x1818247335718912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982220:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335718912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335718912:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019514.982250:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719040:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019514.982270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019514.982277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86dc000 x1818247335719040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335719040/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982312:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719040:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019514.982318:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719168:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019514.982333:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019514.982339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86dd180 x1818247335719168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982349:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335719168/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982363:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719168:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019514.982393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019514.982407:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019514.982414:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86dd500 x1818247335719296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982424:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335719296/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982432:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982434:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982437:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019514.982444:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335719424:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019514.982457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019514.982464:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86df100 x1818247335719424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982474:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335719424/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982481:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982483:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335719424:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019514.982492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335719552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019514.982504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019514.982510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86dc700 x1818247335719552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982519:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335719552/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335719552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019514.982538:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335719680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019514.982552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019514.982558:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019514/real 1734019514] req@ffff8800a86de680 x1818247335719680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019514.982569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335719680/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019514.982577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019514.982579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019514.982582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335719680:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019514.995975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019515.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019515.997968:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019516.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019516.999963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019517.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019518.001964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019518.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019519.003977:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019519.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019519.989835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.989846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019519.989851:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.989862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.989872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019519.989879:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019519.989926:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335719808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019519.989961:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.989966:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019519.989967:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.989972:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.989975:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019519.989978:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019519.989988:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019519.989998:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335719936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019519.990008:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.990011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019519.990012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.990017:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.990019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019519.990023:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019519.990027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019519.990035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335720064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019519.990043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.990046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019519.990048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.990052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.990055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019519.990058:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019519.990063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019519.990073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335720192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019519.990081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.990084:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019519.990086:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.990089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.990092:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019519.990095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019519.990099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019519.990107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335720320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019519.990116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.990119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019519.990121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.990124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.990127:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019519.990127:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335719808:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019519.990130:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019519.990134:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019519.990142:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335720448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019519.990150:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019519.990153:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019519.990155:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019519.990158:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3784 00000100:00080000:2.0:1734019519.990161:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019519.990164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019519.990167:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019519.990175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335720576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019519.990226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86dca80 x1818247335719808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335719808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990295:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990299:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990304:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335719808:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019519.990312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719936:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019519.990331:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990337:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86dd880 x1818247335719936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990357:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335719936/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990366:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990368:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990372:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335719936:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019519.990378:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720064:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019519.990393:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990399:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86df480 x1818247335720064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990409:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335720064/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990416:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990419:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990422:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720064:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019519.990428:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019519.990441:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990447:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86de680 x1818247335720192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335720192/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990464:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990468:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990471:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019519.990504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335720320:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019519.990518:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990524:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86dc700 x1818247335720320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335720320/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990542:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335720320:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019519.990554:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335720448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019519.990566:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990571:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86df100 x1818247335720448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990581:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335720448/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990589:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990591:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990594:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335720448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019519.990600:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335720576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019519.990616:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019519.990622:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019519/real 1734019519] req@ffff8800a86dd500 x1818247335720576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019519.990633:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335720576/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019519.990699:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019519.990700:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019519.990702:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335720576:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019520.005876:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019520.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019521.007989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019521.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019522.009892:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019522.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019523.011921:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019523.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019524.013979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019524.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019524.997829:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.997840:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019524.997845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.997856:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.997865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.997873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019524.997909:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335720704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019524.997945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.997949:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019524.997951:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.997956:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.997959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.997962:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019524.997972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019524.997983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335720832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019524.997992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.997995:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019524.997997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.998001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.998004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.998007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019524.998012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019524.998019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335720960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019524.998028:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.998031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019524.998033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.998036:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.998038:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.998042:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019524.998046:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019524.998056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335721088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019524.998064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.998068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019524.998069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.998073:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.998075:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.998078:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019524.998082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019524.998091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335721216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019524.998093:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335720704:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019524.998100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.998103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019524.998104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.998108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.998110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.998114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019524.998118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019524.998125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335721344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019524.998132:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019524.998135:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019524.998137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019524.998140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3789 00000100:00080000:2.0:1734019524.998143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019524.998146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019524.998150:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019524.998157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335721472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019524.998189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86dd180 x1818247335720704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335720704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998256:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998266:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335720704:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019524.998273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720832:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019524.998295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998303:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86dc000 x1818247335720832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335720832/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998323:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998326:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720832:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019524.998335:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720960:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019524.998350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86ddc00 x1818247335720960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335720960/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998374:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998379:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335720960:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019524.998385:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019524.998398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86df800 x1818247335721088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335721088/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019524.998466:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335721216:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019524.998479:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86de300 x1818247335721216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335721216/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998506:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998509:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335721216:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019524.998515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335721344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019524.998526:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998532:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86dc380 x1818247335721344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998542:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335721344/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998550:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998589:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998593:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335721344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019524.998600:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335721472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019524.998617:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019524.998624:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019524/real 1734019524] req@ffff8800a86dea00 x1818247335721472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019524.998636:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335721472/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019524.998644:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019524.998646:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019524.998649:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335721472:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019525.015952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019525.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019526.017986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019526.213829:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019527.019979:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019527.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019528.021941:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019528.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019529.023989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019529.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019530.005833:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.005844:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019530.005849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.005859:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.005869:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.005877:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019530.005914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335721600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019530.005948:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.005952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019530.005954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.005959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.005962:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.005965:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019530.005976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019530.005987:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335721728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019530.005997:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.006000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019530.006002:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.006006:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.006008:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.006011:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019530.006016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019530.006024:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335721856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019530.006033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.006036:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019530.006037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.006041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.006043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.006047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019530.006051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019530.006061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335721984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019530.006071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.006074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019530.006076:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.006080:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.006082:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.006086:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019530.006091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019530.006095:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335721600:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019530.006099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335722112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019530.006107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.006110:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019530.006111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.006115:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.006117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.006120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019530.006124:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019530.006132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335722240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019530.006139:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019530.006142:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019530.006144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019530.006147:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3794 00000100:00080000:2.0:1734019530.006149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019530.006152:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019530.006157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019530.006164:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335722368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019530.006189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006220:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86dfb80 x1818247335721600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006242:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335721600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335721600:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019530.006273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721728:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019530.006292:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006299:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86ddf80 x1818247335721728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006310:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335721728/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006322:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721728:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019530.006331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721856:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019530.006346:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006352:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86dce00 x1818247335721856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006363:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335721856/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006377:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721856:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019530.006408:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721984:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019530.006424:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86dea00 x1818247335721984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335721984/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006450:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006452:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006455:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335721984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019530.006461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335722112:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019530.006521:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86dc380 x1818247335722112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335722112/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335722112:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019530.006558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335722240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019530.006571:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006577:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86de300 x1818247335722240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006595:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335722240/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335722240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019530.006614:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335722368:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019530.006627:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019530.006632:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019530/real 1734019530] req@ffff8800a86df800 x1818247335722368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019530.006642:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335722368/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019530.006650:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019530.006652:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019530.006655:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335722368:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019530.025965:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019530.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019531.027973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019531.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019532.029980:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019532.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019533.031943:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019533.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019534.033984:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019534.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019535.013838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.013849:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019535.013853:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.013864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.013874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.013882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019535.013919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335722496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019535.013954:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.013958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019535.013960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.013965:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.013968:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.013972:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019535.013982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019535.013993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335722624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019535.014002:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.014005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019535.014008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.014012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.014014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.014018:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019535.014022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019535.014030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335722752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019535.014039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.014042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019535.014044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.014047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.014049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.014053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019535.014057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019535.014066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335722880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019535.014076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.014079:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019535.014080:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.014084:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.014086:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.014090:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335722496:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019535.014094:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019535.014102:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335723008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019535.014111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.014114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019535.014116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.014119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.014121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.014125:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019535.014129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019535.014136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335723136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019535.014144:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019535.014147:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019535.014148:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019535.014151:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3799 00000100:00080000:2.0:1734019535.014169:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019535.014173:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019535.014177:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019535.014183:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019535.014185:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335723264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019535.014213:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86ddc00 x1818247335722496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335722496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014250:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014254:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335722496:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019535.014267:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335722624:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019535.014286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86dc000 x1818247335722624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335722624/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014312:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335722624:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019535.014325:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335722752:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019535.014339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86dd180 x1818247335722752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014355:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335722752/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014370:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335722752:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019535.014432:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335722880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019535.014450:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86dd500 x1818247335722880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014469:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335722880/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014477:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014479:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335722880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019535.014489:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723008:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019535.014502:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014507:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86df100 x1818247335723008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014518:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335723008/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014527:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014529:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014532:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723008:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019535.014537:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019535.014553:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86dc700 x1818247335723136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335723136/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014580:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014583:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019535.014589:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019535.014602:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019535.014608:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019535/real 1734019535] req@ffff8800a86de680 x1818247335723264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019535.014618:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335723264/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019535.014625:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019535.014627:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019535.014630:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723264:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019535.035950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019535.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019536.037942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019536.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019537.039979:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019537.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019538.041959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019538.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019539.043989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019539.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019540.021787:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.021798:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019540.021803:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.021814:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.021823:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.021830:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019540.021867:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335723392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019540.021902:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.021906:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019540.021908:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.021913:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.021916:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.021919:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019540.021929:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019540.021939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335723520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019540.021949:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.021952:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019540.021954:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.021958:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.021960:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.021964:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019540.021968:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019540.021976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335723648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019540.021985:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.021989:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019540.021990:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.021993:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.021996:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.021999:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019540.022003:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019540.022014:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335723776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019540.022022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.022025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019540.022026:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.022030:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.022032:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.022036:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019540.022040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019540.022048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335723904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019540.022051:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335723392:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019540.022057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.022060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019540.022062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.022066:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.022069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.022072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019540.022076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019540.022084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335724032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019540.022091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019540.022094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019540.022096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019540.022099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3804 00000100:00080000:2.0:1734019540.022102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019540.022105:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019540.022108:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019540.022116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335724160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019540.022147:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022177:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86df480 x1818247335723392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022199:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335723392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022212:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022216:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335723392:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019540.022228:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335723520:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019540.022247:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022253:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86dd880 x1818247335723520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022265:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335723520/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022276:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022279:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335723520:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019540.022286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335723648:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019540.022347:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86dca80 x1818247335723648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022366:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335723648/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022381:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335723648:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019540.022387:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335723776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019540.022402:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86de680 x1818247335723776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335723776/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022430:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022433:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335723776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019540.022463:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723904:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019540.022477:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022483:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86dc700 x1818247335723904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335723904/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022501:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335723904:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019540.022515:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019540.022527:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022534:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86df100 x1818247335724032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022544:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335724032/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022551:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022554:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022557:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019540.022562:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019540.022575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019540.022581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019540/real 1734019540] req@ffff8800a86dd500 x1818247335724160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019540.022590:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335724160/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019540.022598:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019540.022600:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019540.022603:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724160:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019540.045945:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019540.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019541.047996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019541.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019542.049992:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019542.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019543.051942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019543.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019544.053996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019544.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019545.029828:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.029839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019545.029844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.029855:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.029865:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.029873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019545.029910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335724288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019545.029945:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.029950:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019545.029952:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.029956:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.029959:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.029963:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019545.029972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019545.029983:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335724416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019545.029993:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.029996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019545.029998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.030001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.030004:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.030007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019545.030012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019545.030020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335724544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019545.030029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.030031:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019545.030033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.030037:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.030039:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.030043:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019545.030047:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019545.030056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335724672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019545.030066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.030069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019545.030070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.030074:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.030076:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.030080:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030083:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335724288:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019545.030084:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019545.030092:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335724800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019545.030100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.030103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019545.030104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.030108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.030110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.030113:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019545.030117:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019545.030124:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335724928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019545.030133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019545.030136:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019545.030137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019545.030141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3809 00000100:00080000:2.0:1734019545.030163:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019545.030167:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019545.030171:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019545.030177:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019545.030179:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335725056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019545.030208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86dd180 x1818247335724288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335724288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030284:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030288:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030294:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335724288:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019545.030303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335724416:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019545.030325:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030333:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86dc000 x1818247335724416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335724416/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030356:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030359:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335724416:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019545.030365:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335724544:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019545.030381:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030387:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86ddc00 x1818247335724544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030399:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335724544/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030407:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030409:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030412:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335724544:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019545.030442:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335724672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019545.030455:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030461:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86df800 x1818247335724672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030471:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335724672/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030479:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030482:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030486:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335724672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019545.030492:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724800:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019545.030504:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030510:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86de300 x1818247335724800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030520:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335724800/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030528:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030532:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030535:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724800:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019545.030541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019545.030554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86dc380 x1818247335724928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335724928/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335724928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019545.030588:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019545.030600:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019545.030606:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019545/real 1734019545] req@ffff8800a86dea00 x1818247335725056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019545.030616:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335725056/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019545.030624:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019545.030626:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019545.030629:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725056:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019545.055926:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019545.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019546.057994:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019546.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019547.059996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019547.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019548.061969:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019548.213820:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019549.063998:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019549.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019550.037839:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.037851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019550.037855:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019550.037866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.037875:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.037882:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019550.037917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335725184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019550.037951:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.037956:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019550.037958:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019550.037963:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.037966:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.037969:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019550.037979:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019550.037989:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335725312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019550.037999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.038002:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019550.038004:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019550.038008:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.038010:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.038014:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019550.038018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019550.038026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335725440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019550.038034:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.038037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019550.038039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019550.038042:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.038045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.038048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019550.038052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019550.038061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335725568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019550.038070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.038073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019550.038075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019550.038078:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.038080:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.038084:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019550.038088:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019550.038095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335725696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019550.038105:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.038107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019550.038109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019550.038109:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335725184:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019550.038113:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.038115:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.038118:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019550.038122:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019550.038171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335725824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019550.038196:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019550.038199:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019550.038200:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019550.038203:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3814 00000100:00080000:2.0:1734019550.038207:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019550.038209:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019550.038213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019550.038222:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335725952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019550.038274:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038304:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86dce00 x1818247335725184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038325:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335725184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038347:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335725184:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019550.038355:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335725312:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019550.038375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86ddf80 x1818247335725312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335725312/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038404:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038408:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335725312:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019550.038423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335725440:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019550.038438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038446:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86dfb80 x1818247335725440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038457:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335725440/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038465:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038467:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038470:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335725440:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019550.038476:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335725568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019550.038489:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038495:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86dea00 x1818247335725568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038504:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335725568/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038512:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038514:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038517:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335725568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019550.038549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725696:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019550.038563:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038570:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86dc380 x1818247335725696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335725696/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038596:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038598:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038602:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725696:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019550.038608:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019550.038620:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038626:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86de300 x1818247335725824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038636:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335725824/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038643:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038646:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038649:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019550.038655:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019550.038667:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019550.038673:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019550/real 1734019550] req@ffff8800a86df800 x1818247335725952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019550.038683:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335725952/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019550.038690:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019550.038693:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019550.038696:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335725952:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019550.065944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019550.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019551.067999:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019551.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019552.069995:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019552.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019553.071986:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019553.213800:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019554.073997:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019554.213825:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019555.045844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.045854:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019555.045859:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.045869:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.045879:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.045886:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019555.045923:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335726080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019555.045958:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.045962:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019555.045964:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.045969:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.045972:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.045976:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019555.045986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019555.045997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335726208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019555.046007:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.046010:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019555.046012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.046015:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.046017:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.046021:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019555.046025:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019555.046033:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335726336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019555.046041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.046073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019555.046075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.046079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.046082:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.046086:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019555.046090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019555.046101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335726464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019555.046111:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.046114:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019555.046116:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.046120:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.046122:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.046126:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019555.046130:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019555.046136:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335726080:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019555.046138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335726592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019555.046146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.046149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019555.046151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.046155:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.046157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.046160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019555.046165:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019555.046172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335726720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019555.046180:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019555.046182:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019555.046184:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019555.046187:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3819 00000100:00080000:2.0:1734019555.046189:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019555.046192:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019555.046196:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019555.046203:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335726848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019555.046262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335726208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019555.046286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335726336:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019555.046314:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335726464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019555.046332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335726592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019555.046350:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335726720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019555.046366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335726848:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019555.075975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019555.213855:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019556.077985:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019556.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019557.079977:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019557.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734019558.052052:0:1957:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019558.052113:0:1957:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019558.052138:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019558.052184:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019558.052195:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019558.052203:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019558.052210:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019558.052217:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019558.052224:0:1957:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019558.052293:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86dd500 x1818247335726464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335726464/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052338:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052342:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052348:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335726464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019558.052357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86df100 x1818247335726592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052371:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335726592/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052379:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052382:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335726592:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019558.052392:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86dc700 x1818247335726720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335726720/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052410:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052414:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335726720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019558.052423:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86de680 x1818247335726848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052444:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335726848/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052452:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335726848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019558.052467:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86ddc00 x1818247335726080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335726080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335726080:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019558.052500:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86dc000 x1818247335726208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335726208/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052521:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052525:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335726208:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019558.052536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019555/real 1734019558] req@ffff8800a86dd180 x1818247335726336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019558.052554:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335726336/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019558.052566:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019558.052569:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019558.052574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335726336:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019558.081940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019558.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019559.084019:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019559.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019560.053800:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.053810:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019560.053814:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.053825:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.053833:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.053841:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019560.053876:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335726976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019560.053910:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.053915:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019560.053917:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.053921:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.053924:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.053928:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019560.053937:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019560.053948:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335727104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019560.053957:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.053960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019560.053992:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.053996:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.053999:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.054003:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019560.054008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019560.054017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335727232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019560.054027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.054030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019560.054032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.054035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.054037:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.054041:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019560.054045:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019560.054054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335727360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019560.054065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.054068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019560.054069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.054073:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.054075:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.054079:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019560.054083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019560.054089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335726976:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019560.054091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335727488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019560.054100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.054103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019560.054105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.054108:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.054110:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.054114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019560.054118:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019560.054125:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335727616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019560.054133:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019560.054136:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019560.054137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019560.054140:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3824 00000100:00080000:2.0:1734019560.054143:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019560.054146:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019560.054149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019560.054157:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335727744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019560.054182:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86dca80 x1818247335726976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335726976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054247:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054257:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335726976:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019560.054264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335727104:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019560.054282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054289:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86dd880 x1818247335727104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335727104/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054309:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054312:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054315:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335727104:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019560.054321:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335727232:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019560.054336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86df480 x1818247335727232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335727232/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054363:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335727232:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019560.054398:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335727360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019560.054411:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054417:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86dd180 x1818247335727360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054427:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335727360/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054435:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054437:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054441:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335727360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019560.054447:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335727488:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019560.054460:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86dc000 x1818247335727488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054475:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335727488/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054483:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054486:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054489:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335727488:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019560.054495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335727616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019560.054507:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054513:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86ddc00 x1818247335727616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054523:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335727616/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054531:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054533:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335727616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019560.054541:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335727744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019560.054554:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019560.054560:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019560/real 1734019560] req@ffff8800a86de680 x1818247335727744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019560.054569:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335727744/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019560.054577:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019560.054579:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019560.054582:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335727744:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019560.086021:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019560.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019561.087991:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019561.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019562.090002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019562.213826:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019563.091991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019563.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019564.094009:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019564.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019565.061848:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.061859:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019565.061864:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.061874:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.061920:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.061929:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019565.061964:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335727872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019565.061998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.062002:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019565.062004:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.062008:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.062011:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.062015:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019565.062025:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019565.062035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335728000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019565.062045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.062048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019565.062050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.062054:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.062057:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.062060:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019565.062064:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019565.062072:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335728128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019565.062082:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.062085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019565.062087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.062090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.062093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.062096:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019565.062100:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019565.062109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335728256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019565.062117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.062120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019565.062122:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.062125:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.062127:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.062131:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019565.062135:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019565.062143:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335728384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019565.062149:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335727872:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019565.062153:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.062156:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019565.062157:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.062161:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.062164:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.062167:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019565.062172:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019565.062179:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335728512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019565.062187:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019565.062189:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019565.062191:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019565.062194:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3829 00000100:00080000:2.0:1734019565.062196:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019565.062199:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019565.062203:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019565.062210:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335728640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019565.062243:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86dc700 x1818247335727872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335727872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062315:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335727872:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019565.062328:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728000:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019565.062347:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062354:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86df100 x1818247335728000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062365:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335728000/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062373:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062376:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728000:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019565.062386:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728128:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019565.062401:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062408:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86dd500 x1818247335728128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062419:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335728128/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062427:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062429:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062432:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728128:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019565.062438:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019565.062450:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062456:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86df800 x1818247335728256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062466:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335728256/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019565.062509:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335728384:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019565.062523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86de300 x1818247335728384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062541:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335728384/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062559:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062562:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335728384:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019565.062568:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335728512:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019565.062580:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062586:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86dc380 x1818247335728512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062596:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335728512/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062608:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335728512:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019565.062614:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335728640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019565.062627:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019565.062633:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019565/real 1734019565] req@ffff8800a86dea00 x1818247335728640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019565.062643:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335728640/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019565.062650:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019565.062653:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019565.062656:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335728640:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019565.095967:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019565.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019566.097989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019566.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019567.100044:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019567.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019568.101989:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019568.213877:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019569.103950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019569.213878:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019570.069857:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.069869:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019570.069875:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.069888:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.069898:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.069908:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019570.069955:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335728768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019570.069992:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.069996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019570.069998:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.070003:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.070006:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.070010:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019570.070022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019570.070034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335728896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019570.070043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.070046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019570.070048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.070052:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.070055:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.070059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019570.070063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019570.070071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335729024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019570.070080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.070083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019570.070084:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.070089:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.070091:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.070095:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019570.070099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019570.070111:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335729152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019570.070112:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335728768:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019570.070120:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.070124:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019570.070125:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.070129:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.070131:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.070135:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019570.070139:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019570.070148:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335729280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019570.070156:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.070159:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019570.070161:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.070164:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.070182:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.070186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019570.070190:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019570.070195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019570.070199:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335729408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019570.070207:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019570.070211:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019570.070212:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019570.070216:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3834 00000100:00080000:2.0:1734019570.070218:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019570.070222:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86dfb80 x1818247335728768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019570.070226:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019570.070234:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335729536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019570.070243:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335728768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070254:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070258:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070263:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335728768:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019570.070270:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728896:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019570.070288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070294:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86ddf80 x1818247335728896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019570.070306:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335728896/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070314:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070317:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335728896:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019570.070330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729024:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019570.070350:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070356:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86dce00 x1818247335729024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019570.070367:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335729024/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070375:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070377:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070381:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729024:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019570.070409:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019570.070422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86dea00 x1818247335729152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019570.070438:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335729152/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070446:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070448:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070452:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019570.070457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335729280:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019570.070469:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86dc380 x1818247335729280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019570.070486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335729280/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335729280:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019570.070504:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335729408:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019570.070515:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070521:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86de300 x1818247335729408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019570.070531:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335729408/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070538:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070541:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070544:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335729408:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019570.070549:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335729536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019570.070562:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019570.070568:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019570/real 1734019570] req@ffff8800a86df800 x1818247335729536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019570.070578:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335729536/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019570.070585:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019570.070588:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019570.070591:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335729536:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019570.105994:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019570.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019571.108020:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019571.213861:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019572.109986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019572.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019573.111975:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019573.213858:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019574.113983:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019574.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019575.077844:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.077856:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019575.077860:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.077871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.077882:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.077889:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019575.077924:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335729664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019575.077959:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.077963:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019575.077965:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.077970:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.077973:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.077977:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019575.077987:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019575.077997:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335729792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019575.078007:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.078010:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019575.078012:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.078016:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.078019:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.078022:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019575.078027:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019575.078034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335729920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019575.078043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.078046:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019575.078048:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.078051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.078053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.078057:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019575.078061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019575.078070:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335730048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019575.078080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.078083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019575.078085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.078088:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.078090:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.078094:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019575.078098:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019575.078101:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335729664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019575.078105:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335730176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019575.078114:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.078116:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019575.078118:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.078121:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.078123:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.078127:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019575.078131:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019575.078138:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335730304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019575.078146:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019575.078149:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019575.078151:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019575.078154:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3839 00000100:00080000:2.0:1734019575.078157:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019575.078160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019575.078164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019575.078172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335730432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019575.078193:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078224:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86dd500 x1818247335729664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335729664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335729664:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019575.078277:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729792:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019575.078295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078302:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86df100 x1818247335729792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335729792/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078325:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729792:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019575.078336:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729920:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019575.078351:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078357:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86dc700 x1818247335729920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078368:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335729920/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078379:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078382:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335729920:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019575.078412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019575.078425:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86de680 x1818247335730048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335730048/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078451:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078454:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078457:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019575.078462:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335730176:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019575.078475:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86ddc00 x1818247335730176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078492:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335730176/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078502:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078505:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335730176:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019575.078511:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335730304:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019575.078523:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078529:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86dc000 x1818247335730304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078539:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335730304/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078547:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078549:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078552:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335730304:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019575.078558:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335730432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019575.078571:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019575.078577:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019575/real 1734019575] req@ffff8800a86dd180 x1818247335730432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019575.078587:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335730432/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019575.078603:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019575.078605:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019575.078609:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335730432:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019575.115966:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019575.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019576.117964:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019576.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019577.119964:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019577.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019578.121948:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019578.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019579.123957:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019579.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019580.085858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.085868:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019580.085872:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.085883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.085892:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.085900:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019580.085938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335730560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019580.085974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.085979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019580.085981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.085985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.085988:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.085992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019580.086002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019580.086013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335730688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019580.086023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.086027:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019580.086028:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.086032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.086034:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.086038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019580.086042:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019580.086050:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335730816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019580.086058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.086062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019580.086063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.086066:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.086069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.086072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019580.086076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019580.086087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335730944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019580.086095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.086098:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019580.086099:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.086103:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.086106:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.086109:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019580.086113:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019580.086121:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335731072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019580.086125:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335730560:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019580.086131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.086134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019580.086136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.086139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.086142:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.086145:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019580.086149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019580.086156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335731200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019580.086164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019580.086167:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019580.086168:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019580.086171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3844 00000100:00080000:2.0:1734019580.086174:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019580.086177:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019580.086181:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019580.086189:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335731328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019580.086219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086250:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86df480 x1818247335730560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086273:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335730560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335730560:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019580.086303:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730688:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019580.086323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086330:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86dd880 x1818247335730688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086341:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335730688/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086361:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086364:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730688:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019580.086371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730816:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019580.086385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86dca80 x1818247335730816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086402:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335730816/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086412:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086415:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730816:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019580.086422:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019580.086435:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86dd180 x1818247335730944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335730944/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086461:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335730944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019580.086495:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731072:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019580.086508:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086514:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86dc000 x1818247335731072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086525:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335731072/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086533:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086539:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731072:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019580.086544:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731200:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019580.086556:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086562:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86ddc00 x1818247335731200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086572:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335731200/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086580:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086582:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731200:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019580.086591:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019580.086603:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019580.086609:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019580/real 1734019580] req@ffff8800a86de680 x1818247335731328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019580.086620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335731328/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019580.086677:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019580.086680:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019580.086684:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731328:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019580.125911:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019580.213759:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019581.127993:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019581.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019582.129944:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019582.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019583.131931:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019583.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019584.133943:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019584.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019585.093858:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.093868:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019585.093873:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.093883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.093893:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.093901:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019585.093938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335731456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019585.093975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.093980:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019585.093981:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.093986:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.093989:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.093993:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019585.094002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019585.094013:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335731584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019585.094023:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.094026:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019585.094029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.094032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.094035:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.094038:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019585.094043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019585.094050:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335731712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019585.094059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.094062:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019585.094064:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.094067:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.094069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.094073:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019585.094077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019585.094086:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335731840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019585.094095:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.094099:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019585.094100:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.094104:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.094106:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.094110:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019585.094114:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019585.094115:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335731456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019585.094122:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335731968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019585.094131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.094134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019585.094136:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.094139:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.094144:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.094147:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019585.094151:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019585.094158:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335732096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019585.094166:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019585.094169:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019585.094170:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019585.094173:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3849 00000100:00080000:2.0:1734019585.094195:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019585.094198:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019585.094202:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019585.094208:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019585.094210:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335732224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019585.094239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86dc700 x1818247335731456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335731456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335731456:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019585.094291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335731584:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019585.094310:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019585.094316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86df100 x1818247335731584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094336:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335731584/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094345:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094348:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094351:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335731584:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019585.094357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335731712:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019585.094372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019585.094378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86dd500 x1818247335731712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335731712/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335731712:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019585.094431:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335731840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019585.094444:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019585.094451:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86df800 x1818247335731840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094462:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335731840/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094470:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094473:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094477:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335731840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019585.094483:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731968:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019585.094496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019585.094502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86de300 x1818247335731968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094512:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335731968/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094521:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094523:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094526:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335731968:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019585.094532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732096:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019585.094590:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019585.094599:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86dc380 x1818247335732096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094609:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335732096/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094617:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094620:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094623:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732096:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019585.094630:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019585.094649:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019585.094655:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019585/real 1734019585] req@ffff8800a86dea00 x1818247335732224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019585.094667:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335732224/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019585.094674:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019585.094676:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019585.094679:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732224:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019585.135972:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019585.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019586.138003:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019586.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019587.139942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019587.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019588.141999:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019588.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019589.143965:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019589.213775:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019590.101840:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.101853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019590.101858:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.101872:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.101884:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019590.101894:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019590.101939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335732352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019590.101979:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.101983:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019590.101985:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.101990:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.101994:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019590.101997:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019590.102009:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019590.102021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335732480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019590.102031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.102034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019590.102037:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.102040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.102043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019590.102047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019590.102051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019590.102060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335732608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019590.102070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.102073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019590.102075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.102079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.102081:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019590.102085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019590.102089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019590.102096:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335732352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019590.102100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335732736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019590.102109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.102112:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019590.102114:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.102117:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.102120:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019590.102124:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019590.102128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019590.102136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335732864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019590.102145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.102148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019590.102150:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.102154:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.102167:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019590.102172:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019590.102176:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019590.102179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019590.102184:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335732992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019590.102192:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019590.102195:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019590.102197:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019590.102200:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3854 00000100:00080000:2.0:1734019590.102203:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019590.102206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86dce00 x1818247335732352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019590.102207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019590.102211:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019590.102219:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335733120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019590.102227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335732352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102242:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102247:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335732352:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019590.102255:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335732480:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019590.102275:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019590.102282:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86ddf80 x1818247335732480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019590.102293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335732480/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102301:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102304:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335732480:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019590.102313:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335732608:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019590.102326:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019590.102332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86dfb80 x1818247335732608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019590.102342:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335732608/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335732608:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019590.102383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335732736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019590.102396:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019590.102403:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86dea00 x1818247335732736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019590.102422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335732736/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102430:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102432:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335732736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019590.102440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732864:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019590.102451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019590.102501:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86dc380 x1818247335732864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019590.102514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335732864/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102524:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102527:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732864:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019590.102533:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732992:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019590.102552:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019590.102559:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86de300 x1818247335732992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019590.102570:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335732992/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102581:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102585:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335732992:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019590.102590:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335733120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019590.102604:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019590.102610:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019590/real 1734019590] req@ffff8800a86df800 x1818247335733120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019590.102620:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335733120/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019590.102628:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019590.102630:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019590.102633:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335733120:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019590.145954:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019590.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019591.147990:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019591.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019592.149996:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019592.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019593.151962:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019593.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019594.153950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019594.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019595.109838:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.109847:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019595.109852:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.109862:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.109872:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.109881:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019595.109918:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335733248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019595.109953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.109957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019595.109959:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.109963:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.109967:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.109970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019595.109980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019595.109990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335733376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019595.110000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.110003:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019595.110005:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.110008:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.110011:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.110014:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019595.110018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019595.110026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335733504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019595.110035:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.110037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019595.110039:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.110043:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.110046:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.110049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019595.110053:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019595.110063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335733632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019595.110071:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.110074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019595.110076:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.110079:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.110081:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.110085:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019595.110089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019595.110097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335733760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019595.110100:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335733248:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019595.110106:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.110109:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019595.110111:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.110115:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.110117:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.110120:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019595.110125:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019595.110132:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335733888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019595.110141:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019595.110144:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019595.110145:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019595.110148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3859 00000100:00080000:2.0:1734019595.110151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019595.110154:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019595.110158:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019595.110165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335734016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019595.110195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86dd500 x1818247335733248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110247:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335733248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110260:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110265:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110271:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335733248:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019595.110279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335733376:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019595.110298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86df100 x1818247335733376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110315:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335733376/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110327:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335733376:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019595.110337:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335733504:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019595.110356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110362:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86dc700 x1818247335733504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110414:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335733504/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110426:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335733504:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019595.110437:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335733632:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019595.110457:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110465:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86de680 x1818247335733632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110477:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335733632/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110485:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110488:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110491:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335733632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019595.110520:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335733760:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019595.110534:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110540:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86ddc00 x1818247335733760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110551:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335733760/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110558:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110561:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110564:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335733760:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019595.110570:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335733888:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019595.110582:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110588:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86dc000 x1818247335733888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110605:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335733888/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110613:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110616:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110619:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335733888:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019595.110624:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734016:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019595.110636:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019595.110642:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019595/real 1734019595] req@ffff8800a86dd180 x1818247335734016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019595.110652:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335734016/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019595.110660:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019595.110662:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019595.110665:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734016:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019595.155963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019595.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019596.158005:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019596.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019597.159863:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019597.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019598.161960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019598.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019599.163937:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019599.213862:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019600.117847:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.117858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019600.117863:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.117874:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.117883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.117890:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019600.117927:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335734144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019600.117962:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.117966:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019600.117969:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.117974:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.117977:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.117980:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019600.117990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019600.118000:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335734272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019600.118010:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.118013:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019600.118015:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.118019:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.118021:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.118025:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019600.118029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019600.118037:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335734400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019600.118045:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.118048:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019600.118050:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.118053:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.118056:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.118059:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019600.118063:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019600.118073:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335734528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019600.118083:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.118087:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019600.118088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.118092:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.118094:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.118097:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019600.118101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019600.118107:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335734144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019600.118109:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335734656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019600.118117:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.118120:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019600.118121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.118125:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.118128:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.118131:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019600.118135:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019600.118141:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335734784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019600.118149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019600.118152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019600.118154:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019600.118158:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3864 00000100:00080000:2.0:1734019600.118161:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019600.118164:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019600.118168:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019600.118175:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335734912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019600.118199:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118231:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86dca80 x1818247335734144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335734144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118268:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118272:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118278:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335734144:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019600.118286:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335734272:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019600.118340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118349:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86dd880 x1818247335734272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118362:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335734272/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118370:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118373:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118376:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335734272:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019600.118383:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335734400:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019600.118398:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118404:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86df480 x1818247335734400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335734400/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118423:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118427:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118430:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335734400:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019600.118461:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335734528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019600.118475:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118482:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86dd180 x1818247335734528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118493:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335734528/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118500:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118503:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118506:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335734528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019600.118512:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734656:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019600.118524:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118530:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86dc000 x1818247335734656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118540:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335734656/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118548:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118550:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118553:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734656:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019600.118559:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734784:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019600.118575:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118581:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86ddc00 x1818247335734784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118592:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335734784/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118600:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118602:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118605:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734784:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019600.118610:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019600.118623:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019600.118629:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019600/real 1734019600] req@ffff8800a86de680 x1818247335734912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019600.118639:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335734912/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019600.118646:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019600.118649:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019600.118651:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335734912:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019600.165935:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019600.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019601.167944:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019601.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019602.170070:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019602.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019603.171981:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019603.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019604.173971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019604.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019605.125841:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.125851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019605.125856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.125866:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.125880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.125888:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019605.125922:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335735040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019605.125955:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.125960:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019605.125962:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.125966:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.125969:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.125973:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019605.125982:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019605.125993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335735168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019605.126003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.126007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019605.126008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.126012:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.126014:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.126017:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019605.126022:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019605.126030:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335735296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019605.126039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.126042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019605.126043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.126047:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.126049:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.126053:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019605.126056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019605.126066:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335735424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019605.126074:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.126077:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019605.126078:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.126082:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.126085:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.126088:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019605.126093:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019605.126101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335735552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019605.126102:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335735040:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019605.126110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.126113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019605.126115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.126118:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.126120:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.126124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019605.126128:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019605.126135:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335735680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019605.126143:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019605.126146:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019605.126147:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019605.126150:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3869 00000100:00080000:2.0:1734019605.126153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019605.126156:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019605.126160:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019605.126167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335735808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019605.126197:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86dc700 x1818247335735040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126281:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335735040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126294:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126298:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126303:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335735040:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019605.126311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335735168:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019605.126332:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126339:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86df100 x1818247335735168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335735168/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126358:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335735168:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019605.126371:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335735296:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019605.126385:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126391:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86dd500 x1818247335735296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126401:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335735296/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126411:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126413:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126417:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335735296:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019605.126423:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335735424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019605.126435:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126441:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86df800 x1818247335735424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335735424/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126464:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335735424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019605.126496:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335735552:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019605.126509:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126516:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86de300 x1818247335735552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126527:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335735552/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126537:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335735552:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019605.126547:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335735680:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019605.126560:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126566:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86dc380 x1818247335735680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126576:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335735680/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126584:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126586:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126589:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335735680:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019605.126594:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335735808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019605.126606:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019605.126612:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019605/real 1734019605] req@ffff8800a86dea00 x1818247335735808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019605.126623:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335735808/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019605.126631:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019605.126633:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019605.126636:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335735808:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019605.175984:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019605.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019606.177870:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019606.213827:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019607.179912:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019607.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019608.181946:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019608.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019609.183965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019609.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019610.133822:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.133832:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019610.133837:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.133847:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.133856:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.133864:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019610.133901:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335735936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019610.133936:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.133940:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019610.133942:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.133947:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.133950:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.133953:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019610.133963:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019610.133973:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335736064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019610.133983:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.133986:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019610.133988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.133992:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.133995:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.133998:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019610.134003:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019610.134011:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335736192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019610.134020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.134023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019610.134024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.134028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.134030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.134034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019610.134038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019610.134048:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335736320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019610.134057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.134060:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019610.134062:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.134065:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.134067:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.134071:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019610.134075:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1734019610.134080:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335735936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019610.134083:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335736448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019610.134092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.134095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019610.134096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.134100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.134102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.134105:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019610.134110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019610.134116:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335736576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019610.134160:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019610.134163:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019610.134165:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019610.134168:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3874 00000100:00080000:2.0:1734019610.134171:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019610.134174:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019610.134178:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019610.134187:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335736704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019610.134215:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134247:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86dfb80 x1818247335735936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335735936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134286:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335735936:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019610.134300:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736064:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019610.134320:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86ddf80 x1818247335736064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134339:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335736064/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134347:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134350:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134354:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736064:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019610.134360:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736192:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019610.134375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86dce00 x1818247335736192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134392:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335736192/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736192:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019610.134436:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019610.134449:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134455:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86dea00 x1818247335736320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134465:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335736320/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019610.134485:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335736448:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019610.134498:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134504:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86dc380 x1818247335736448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335736448/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134526:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134529:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335736448:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019610.134535:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335736576:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019610.134548:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134554:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86de300 x1818247335736576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134563:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335736576/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134571:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134573:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134576:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335736576:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019610.134582:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335736704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019610.134595:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019610.134600:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019610/real 1734019610] req@ffff8800a86df800 x1818247335736704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019610.134610:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335736704/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019610.134618:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019610.134621:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019610.134624:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335736704:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019610.185939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019610.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019611.187982:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019611.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019612.190007:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019612.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019613.191969:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019613.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019614.193942:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019614.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019615.141816:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.141827:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019615.141832:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.141842:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.141851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.141859:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019615.141894:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335736832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019615.141928:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.141932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019615.141934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.141939:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.141942:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.141946:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019615.141955:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019615.141965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335736960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019615.141975:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.141978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019615.141980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.141983:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.141986:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.141989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019615.141993:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019615.142001:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335737088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019615.142011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.142014:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019615.142016:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.142019:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.142021:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.142025:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019615.142029:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019615.142080:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335737216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019615.142091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.142094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019615.142096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.142100:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.142102:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.142106:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142108:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335736832:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019615.142110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019615.142119:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335737344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019615.142127:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.142130:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019615.142132:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.142135:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.142141:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.142144:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019615.142149:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019615.142156:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335737472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019615.142164:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019615.142166:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019615.142168:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019615.142171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3879 00000100:00080000:2.0:1734019615.142192:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019615.142195:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019615.142199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019615.142206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019615.142207:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335737600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019615.142237:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86dd500 x1818247335736832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142261:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335736832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142274:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142278:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335736832:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019615.142291:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736960:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019615.142310:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142317:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86df100 x1818247335736960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142328:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335736960/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142339:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142343:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335736960:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019615.142349:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737088:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019615.142364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142371:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86dc700 x1818247335737088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142382:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335737088/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142390:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142392:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737088:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019615.142425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019615.142438:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142444:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86de680 x1818247335737216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142454:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335737216/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142461:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142464:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142467:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019615.142473:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335737344:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019615.142486:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142492:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86ddc00 x1818247335737344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142502:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335737344/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142518:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142520:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142524:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335737344:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019615.142529:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335737472:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019615.142541:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142547:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86dc000 x1818247335737472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142556:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335737472/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142564:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142566:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142570:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335737472:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019615.142576:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335737600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019615.142589:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019615.142594:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019615/real 1734019615] req@ffff8800a86dd180 x1818247335737600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019615.142604:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335737600/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019615.142612:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019615.142614:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019615.142617:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335737600:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019615.195965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019615.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019616.197987:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019616.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019617.199975:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019617.213832:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019618.202024:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019618.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019619.204018:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019619.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019620.149837:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.149851:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019620.149856:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.149871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.149882:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.149891:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019620.149939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335737728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019620.150026:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.150030:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019620.150032:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.150037:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.150041:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.150046:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019620.150057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019620.150071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335737856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019620.150081:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.150085:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019620.150087:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.150090:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.150093:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.150097:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019620.150101:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019620.150110:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335737984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019620.150118:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.150121:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019620.150123:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.150127:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.150130:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.150133:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019620.150138:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019620.150148:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335737728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019620.150150:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335738112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019620.150159:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.150162:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019620.150164:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.150168:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.150172:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.150176:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019620.150180:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019620.150189:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335738240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019620.150196:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.150200:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019620.150201:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.150205:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.150222:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.150226:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019620.150230:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019620.150238:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335738368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019620.150247:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019620.150251:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019620.150252:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019620.150256:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3884 00000100:00080000:2.0:1734019620.150258:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019620.150262:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019620.150264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737856:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734019620.150266:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019620.150274:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335738496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019620.150288:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737984:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019620.150315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335738112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019620.150331:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335738240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019620.150347:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335738368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019620.150361:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335738496:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019620.205991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019620.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019621.208023:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019621.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019622.210011:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019622.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734019623.156087:0:1958:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019623.156142:0:1958:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019623.156167:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019623.156212:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019623.156224:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019623.156236:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019623.156247:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019623.156258:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019623.156268:0:1958:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019623.156348:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86dd180 x1818247335738112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.156389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335738112/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.156409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.156417:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.156425:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335738112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019623.156438:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86dc000 x1818247335738240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.156460:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335738240/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.156473:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.156478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.156485:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335738240:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019623.156496:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86ddc00 x1818247335738368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.156514:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335738368/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.156526:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.156531:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.156536:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335738368:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019623.156546:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86de680 x1818247335738496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.156565:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335738496/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.156578:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.156582:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.156587:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335738496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019623.156603:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86df480 x1818247335737728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.156624:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335737728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.156636:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.156640:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.156645:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335737728:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019623.156657:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86dd880 x1818247335737856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.156675:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335737856/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.156688:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.157016:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.157023:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737856:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019623.157035:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019620/real 1734019623] req@ffff8800a86dca80 x1818247335737984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019623.157057:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335737984/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019623.157069:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019623.157074:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019623.157080:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335737984:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019623.211991:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019623.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734019624.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019624.213987:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019625.157814:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.157829:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019625.157835:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.157848:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00080000:2.0:1734019625.157860:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.157929:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019625.157972:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335738624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019625.158012:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.158016:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019625.158020:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.158024:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00080000:2.0:1734019625.158028:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.158033:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019625.158043:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019625.158054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335738752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019625.158064:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.158068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019625.158069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.158073:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00080000:2.0:1734019625.158075:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.158079:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019625.158083:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019625.158091:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335738880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019625.158099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.158102:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019625.158103:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.158107:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00080000:2.0:1734019625.158109:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.158112:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019625.158116:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019625.158126:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335739008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019625.158132:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335738624:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019625.158136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.158139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019625.158140:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.158144:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00080000:2.0:1734019625.158147:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.158150:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019625.158154:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019625.158162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335739136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019625.158169:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.158172:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019625.158174:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.158177:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00080000:2.0:1734019625.158197:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.158200:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019625.158204:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019625.158206:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019625.158212:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335739264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019625.158221:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019625.158224:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019625.158225:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019625.158228:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3889 00000100:00100000:3.0:1734019625.158230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86dc700 x1818247335738624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019625.158232:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019625.158235:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019625.158239:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019625.158244:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335738624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734019625.158247:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335739392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019625.158253:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158259:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335738624:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019625.158264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335738752:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019625.158276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019625.158280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86df100 x1818247335738752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019625.158287:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335738752/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019625.158292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158293:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335738752:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019625.158299:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335738880:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019625.158307:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019625.158310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86dd500 x1818247335738880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019625.158317:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335738880/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019625.158322:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158324:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335738880:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019625.158344:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019625.158352:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019625.158355:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86df800 x1818247335739008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019625.158360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335739008/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019625.158364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158366:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158368:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019625.158372:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335739136:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019625.158379:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019625.158382:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86de300 x1818247335739136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019625.158387:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335739136/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019625.158392:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158394:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158396:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335739136:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019625.158400:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335739264:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019625.158407:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019625.158410:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86dc380 x1818247335739264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019625.158415:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335739264/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019625.158419:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158420:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158421:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335739264:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019625.158425:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335739392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019625.158432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019625.158435:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019625/real 1734019625] req@ffff8800a86dea00 x1818247335739392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019625.158440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335739392/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019625.158445:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019625.158446:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019625.158448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335739392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019625.213812:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019625.215909:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019626.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019626.217950:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019627.213803:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019627.219920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019628.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019628.221903:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019629.213779:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019629.223854:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019630.165821:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165829:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019630.165832:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.165839:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.165845:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.165850:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019630.165876:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335739520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019630.165897:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165900:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019630.165901:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.165903:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.165905:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.165907:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019630.165913:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019630.165919:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335739648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019630.165924:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165926:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019630.165927:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.165928:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.165930:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.165932:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019630.165934:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019630.165938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335739776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019630.165942:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165944:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019630.165944:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.165946:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.165947:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.165949:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019630.165951:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019630.165958:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335739904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019630.165962:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165964:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019630.165965:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.165967:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.165968:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.165970:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019630.165972:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019630.165976:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335740032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019630.165980:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019630.165982:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.165984:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.165985:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.165987:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019630.165989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019630.165992:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335740160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019630.165997:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019630.165998:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019630.165999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019630.166001:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3894 00000100:00080000:2.0:1734019630.166002:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019630.166004:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019630.166006:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019630.166006:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335739520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019630.166010:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335740288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019630.166068:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166087:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86dce00 x1818247335739520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166098:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335739520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166104:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166106:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166109:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335739520:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019630.166112:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739648:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019630.166122:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166125:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86ddf80 x1818247335739648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166131:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335739648/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166134:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166136:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166137:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739648:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019630.166140:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739776:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019630.166148:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166151:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86dfb80 x1818247335739776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166157:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335739776/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166160:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166161:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166162:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739776:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019630.166165:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019630.166172:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166174:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86dea00 x1818247335739904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166178:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335739904/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166182:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166183:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166184:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335739904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019630.166188:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740032:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019630.166195:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166198:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86dc380 x1818247335740032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166202:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335740032/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166209:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166210:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166212:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740032:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019630.166231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740160:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019630.166237:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166240:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86de300 x1818247335740160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166245:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335740160/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166249:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166250:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166251:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740160:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019630.166254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019630.166260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019630.166262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019630/real 1734019630] req@ffff8800a86df800 x1818247335740288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019630.166267:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335740288/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019630.166270:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019630.166271:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019630.166273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019630.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019630.225879:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019631.213818:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019631.227902:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019632.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019632.230013:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019633.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019633.231869:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019634.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019634.233919:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019635.173809:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.173819:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019635.173824:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.173834:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.173843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019635.173851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019635.173886:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335740416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019635.173921:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.173925:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019635.173927:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.173931:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.173934:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019635.173937:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019635.173947:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019635.173957:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335740544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019635.173966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.173969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019635.173972:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.173975:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.173978:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019635.173981:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019635.173986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019635.173994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335740672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019635.174003:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.174005:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019635.174007:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.174010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.174013:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019635.174016:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019635.174020:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019635.174029:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335740800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019635.174038:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.174041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1734019635.174041:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335740416:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019635.174043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.174047:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.174049:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019635.174052:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019635.174056:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019635.174064:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335740928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019635.174073:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.174076:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019635.174077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.174081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.174083:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019635.174086:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019635.174090:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019635.174097:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335741056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019635.174104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019635.174107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019635.174108:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019635.174111:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3899 00000100:00080000:2.0:1734019635.174114:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734019635.174115:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019635.174117:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019635.174121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019635.174128:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335741184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019635.174137:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86dd500 x1818247335740416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174151:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335740416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174159:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174162:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174165:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335740416:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019635.174169:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335740544:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019635.174180:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019635.174184:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86df100 x1818247335740544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335740544/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174195:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174197:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174198:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335740544:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019635.174204:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335740672:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019635.174214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019635.174218:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86dc700 x1818247335740672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174224:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335740672/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174228:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174230:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174231:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335740672:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019635.174253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335740800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019635.174260:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019635.174264:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86dca80 x1818247335740800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335740800/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335740800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019635.174281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740928:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019635.174288:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019635.174291:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86dd880 x1818247335740928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174297:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335740928/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174302:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174303:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174305:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335740928:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019635.174308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741056:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019635.174315:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019635.174318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86df480 x1818247335741056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335741056/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741056:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019635.174333:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019635.174340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019635.174343:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019635/real 1734019635] req@ffff8800a86de680 x1818247335741184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019635.174348:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335741184/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019635.174352:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019635.174355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019635.174356:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019635.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019635.235956:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019636.213947:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019636.237932:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019637.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019637.239951:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019638.213833:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019638.242003:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019639.213805:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019639.243925:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019640.181835:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.181845:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019640.181849:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.181859:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.181867:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.181873:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019640.181910:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335741312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019640.181936:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.181939:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019640.181940:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.181943:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.181945:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.181947:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019640.181955:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019640.181961:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335741440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019640.181966:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.181969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019640.181970:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.181972:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.181973:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.181975:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019640.181978:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019640.181982:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335741568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019640.181988:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.181990:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019640.181991:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.181994:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.181995:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.181997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019640.181997:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335741312:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019640.181999:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019640.182006:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335741696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019640.182010:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.182012:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019640.182013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.182015:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.182017:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.182019:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019640.182021:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019640.182026:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335741824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019640.182031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.182032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019640.182033:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.182035:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.182051:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.182054:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019640.182057:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019640.182062:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335741952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019640.182062:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019640.182066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019640.182068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019640.182069:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019640.182071:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3904 00000100:00080000:2.0:1734019640.182072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019640.182074:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019640.182076:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019640.182081:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335742080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019640.182085:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86ddc00 x1818247335741312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182101:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335741312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182110:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182113:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182118:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335741312:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019640.182125:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335741440:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019640.182144:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019640.182151:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86dc000 x1818247335741440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182161:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335741440/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182169:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182171:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182174:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335741440:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019640.182196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335741568:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019640.182209:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019640.182216:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86dd180 x1818247335741568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335741568/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182234:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182236:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335741568:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019640.182244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335741696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019640.182257:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019640.182262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86de680 x1818247335741696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182272:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335741696/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182280:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335741696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019640.182287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741824:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019640.182299:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019640.182305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86df480 x1818247335741824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335741824/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182321:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182323:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182325:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741824:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019640.182330:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741952:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019640.182340:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019640.182345:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86dd880 x1818247335741952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182354:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335741952/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182361:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182363:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335741952:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019640.182370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019640.182382:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019640.182388:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019640/real 1734019640] req@ffff8800a86dca80 x1818247335742080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019640.182397:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335742080/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019640.182409:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019640.182411:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019640.182414:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019640.213838:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019640.245924:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019641.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019641.247912:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019642.213824:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019642.249942:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019643.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019643.251929:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019644.213794:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019644.253972:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019645.189874:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.189884:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019645.189888:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.189898:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00080000:2.0:1734019645.189906:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.189913:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019645.189948:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335742208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019645.189972:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.189974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019645.189975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.189978:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00080000:2.0:1734019645.189980:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.189981:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019645.189990:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019645.189995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335742336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019645.190000:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.190002:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019645.190003:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.190005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00080000:2.0:1734019645.190006:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.190008:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019645.190010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019645.190015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335742464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019645.190019:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.190021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019645.190021:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.190024:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00100000:3.0:1734019645.190025:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335742208:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019645.190026:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.190028:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019645.190030:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019645.190037:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335742592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019645.190041:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.190042:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019645.190043:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.190045:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00080000:2.0:1734019645.190046:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.190048:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019645.190050:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019645.190055:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335742720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019645.190059:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.190061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019645.190061:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.190063:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00080000:2.0:1734019645.190064:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.190066:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019645.190068:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019645.190072:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335742848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019645.190076:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019645.190078:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019645.190079:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019645.190081:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3909 00000100:00080000:2.0:1734019645.190082:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019645.190083:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019645.190085:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019645.190090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335742976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019645.190090:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190111:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86dc700 x1818247335742208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190125:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335742208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190134:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190136:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190140:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335742208:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019645.190145:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335742336:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019645.190161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86df100 x1818247335742336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190175:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335742336/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190187:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190189:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190191:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335742336:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019645.190213:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335742464:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019645.190226:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86dd500 x1818247335742464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190239:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335742464/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190246:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190247:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190250:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335742464:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019645.190254:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335742592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019645.190263:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190268:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86df800 x1818247335742592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190275:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335742592/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190283:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190285:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335742592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019645.190290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742720:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019645.190300:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86de300 x1818247335742720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190313:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335742720/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190319:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190320:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190322:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742720:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019645.190326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742848:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019645.190336:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190342:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86dc380 x1818247335742848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190350:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335742848/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190357:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190359:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190361:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742848:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019645.190366:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019645.190377:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019645.190381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019645/real 1734019645] req@ffff8800a86dea00 x1818247335742976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019645.190389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335742976/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019645.190396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019645.190411:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019645.190413:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335742976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019645.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019645.255910:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019646.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019646.257913:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019647.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019647.259922:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019648.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019648.261937:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019649.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019649.263891:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019650.197822:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.197829:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019650.197832:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.197839:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.197846:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.197851:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019650.197876:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335743104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019650.197898:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.197900:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019650.197902:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.197905:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.197906:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.197908:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019650.197914:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019650.197920:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335743232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019650.197925:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.197926:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019650.197927:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.197929:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.197931:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.197932:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019650.197935:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019650.197939:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335743360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019650.197943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.197944:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019650.197945:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.197947:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.197948:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.197950:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019650.197952:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019650.197958:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335743488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019650.197963:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.197965:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019650.197966:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.197968:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.197970:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.197971:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019650.197971:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335743104:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019650.197973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019650.197977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335743616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019650.197981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.197983:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019650.197984:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.197986:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.197987:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.197989:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019650.197991:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019650.197995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335743744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019650.197999:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019650.198000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019650.198001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019650.198002:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3914 00000100:00080000:2.0:1734019650.198010:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019650.198013:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019650.198015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019650.198016:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019650.198019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335743872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019650.198032:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86dfb80 x1818247335743104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198042:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335743104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198048:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198050:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198053:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335743104:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019650.198057:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335743232:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019650.198065:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019650.198069:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86ddf80 x1818247335743232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198079:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335743232/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198083:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198084:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198086:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335743232:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019650.198089:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335743360:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019650.198096:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019650.198099:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86dce00 x1818247335743360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198104:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335743360/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198107:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198109:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198110:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335743360:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019650.198125:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335743488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019650.198131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019650.198135:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86dea00 x1818247335743488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198140:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335743488/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198144:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198145:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198147:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335743488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019650.198150:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335743616:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019650.198156:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019650.198159:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86dc380 x1818247335743616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198164:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335743616/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198168:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198169:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198171:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335743616:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019650.198173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335743744:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019650.198179:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019650.198182:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86de300 x1818247335743744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335743744/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198191:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198192:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198194:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335743744:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019650.198197:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335743872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019650.198205:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019650.198208:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019650/real 1734019650] req@ffff8800a86df800 x1818247335743872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019650.198213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335743872/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019650.198216:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019650.198217:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019650.198219:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335743872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019650.213807:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019650.265832:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019651.213872:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019651.267915:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019652.213846:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019652.269939:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019653.213868:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019653.271870:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019654.213828:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019654.273859:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019655.205833:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.205841:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019655.205845:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.205853:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.205861:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.205867:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019655.205914:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335744000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019655.205940:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.205943:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019655.205944:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.205946:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.205948:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.205950:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019655.205957:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019655.205965:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335744128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019655.205970:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.205972:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019655.205972:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.205974:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.205976:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.205977:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019655.205980:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019655.205984:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335744256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019655.205987:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.205989:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019655.205989:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.205991:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.205992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.205994:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019655.205996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019655.206003:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335744384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019655.206006:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.206008:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019655.206009:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.206011:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.206012:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.206014:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019655.206016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019655.206020:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335744512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019655.206025:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.206026:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019655.206027:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.206029:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.206030:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.206032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019655.206034:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019655.206039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335744640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019655.206041:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335744000:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019655.206042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019655.206044:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019655.206044:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019655.206046:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3919 00000100:00080000:2.0:1734019655.206047:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019655.206049:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019655.206051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019655.206056:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335744768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019655.206095:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206112:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86dd500 x1818247335744000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206123:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335744000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206129:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206132:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206135:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335744000:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019655.206139:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335744128:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019655.206150:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206154:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86df100 x1818247335744128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206160:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335744128/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206164:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206165:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206167:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335744128:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019655.206170:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335744256:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019655.206178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206181:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86dc700 x1818247335744256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206186:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335744256/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206190:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206193:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335744256:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019655.206196:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335744384:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019655.206203:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86dca80 x1818247335744384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206215:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335744384/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206219:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206220:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206222:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335744384:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019655.206239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335744512:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019655.206246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206249:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86dd880 x1818247335744512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335744512/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206260:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335744512:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019655.206264:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335744640:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019655.206270:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206274:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86df480 x1818247335744640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206279:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335744640/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206282:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206284:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206286:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335744640:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019655.206289:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335744768:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019655.206295:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019655.206298:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019655/real 1734019655] req@ffff8800a86de680 x1818247335744768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019655.206303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335744768/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019655.206307:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019655.206308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019655.206310:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335744768:192.168.201.104@tcp:8:kworker.0 00000100:00100000:2.0:1734019655.213744:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019655.275897:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019656.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019656.277863:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019657.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019657.279933:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019658.213819:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019658.281874:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019659.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019659.283868:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1734019660.213842:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019660.213855:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019660.213861:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019660.213874:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00080000:2.0:1734019660.213885:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.213895:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1734019660.213920:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00100000:2.0:1734019660.213941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335744896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019660.213974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019660.213978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019660.213980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019660.213985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00080000:2.0:1734019660.213988:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.213991:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019660.214002:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019660.214012:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335745024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019660.214022:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019660.214025:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019660.214029:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019660.214033:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00080000:2.0:1734019660.214036:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.214040:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019660.214044:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019660.214052:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335745152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019660.214061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019660.214064:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019660.214065:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019660.214069:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00080000:2.0:1734019660.214071:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.214075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019660.214079:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019660.214081:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335744896:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019660.214090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335745280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019660.214099:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019660.214103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019660.214104:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019660.214108:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00080000:2.0:1734019660.214111:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.214114:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019660.214119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019660.214127:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335745408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019660.214136:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1734019660.214137:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019660.214139:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019660.214141:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019660.214144:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00080000:2.0:1734019660.214147:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.214151:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019660.214155:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019660.214155:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86dd180 x1818247335744896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734019660.214162:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335745536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019660.214167:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335744896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019660.214170:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019660.214173:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1734019660.214174:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734019660.214175:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1734019660.214176:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:2.0:1734019660.214178:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3924 00000100:00100000:3.0:1734019660.214180:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335744896:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019660.214181:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019660.214184:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214185:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745024:192.168.201.104@tcp:38:kworker.0 00000008:00100000:2.0:1734019660.214188:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019660.214197:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335745664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019660.214197:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214200:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86dc000 x1818247335745024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019660.214206:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335745024/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019660.214211:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019660.214213:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019660.214215:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745024:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019660.214219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745152:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019660.214228:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214231:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86ddc00 x1818247335745152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019660.214236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335745152/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019660.214240:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019660.214241:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019660.214243:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745152:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019660.214260:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745280:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019660.214269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214273:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86de680 x1818247335745280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019660.214304:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335745280/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019660.214308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019660.214309:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019660.214311:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745280:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019660.214315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335745408:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019660.214323:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214327:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86df480 x1818247335745408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019660.214332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335745408/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019660.214336:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019660.214338:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019660.214339:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335745408:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019660.214342:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335745536:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019660.214348:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214351:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86dd880 x1818247335745536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019660.214360:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335745536/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019660.214364:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019660.214365:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019660.214367:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335745536:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019660.214370:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335745664:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019660.214376:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019660.214379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019660/real 1734019660] req@ffff8800a86dca80 x1818247335745664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019660.214384:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335745664/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019660.214388:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019660.214389:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019660.214390:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335745664:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019660.285923:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019661.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019661.287910:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019662.213918:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019662.289903:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019663.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019663.291900:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019664.213785:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019664.293945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019665.213806:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019665.221799:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.221812:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019665.221816:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019665.221828:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.221838:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019665.221847:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019665.221891:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335745792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019665.221927:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.221932:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019665.221933:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019665.221938:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.221941:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019665.221945:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019665.221957:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019665.221968:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335745920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019665.221978:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.221981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019665.221983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019665.221987:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.221989:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019665.221993:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019665.221997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019665.222006:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335746048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019665.222012:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335745792:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019665.222017:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.222020:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019665.222022:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019665.222026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.222028:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019665.222031:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019665.222036:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019665.222046:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335746176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019665.222054:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.222058:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019665.222059:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019665.222063:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.222066:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734019665.222068:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019665.222070:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019665.222074:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019665.222082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335746304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019665.222086:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86dc700 x1818247335745792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019665.222091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.222094:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019665.222095:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019665.222098:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335745792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019665.222099:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.222102:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1734019665.222105:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1734019665.222106:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734019665.222107:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734019665.222110:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019665.222110:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335745792:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019665.222115:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734019665.222118:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335746432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019665.222126:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019665.222127:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019665.222130:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1734019665.222130:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86df100 x1818247335745920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019665.222131:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019665.222134:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3929 00000100:00080000:2.0:1734019665.222137:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019665.222137:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335745920/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019665.222141:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734019665.222141:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019665.222142:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000008:00100000:2.0:1734019665.222145:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019665.222145:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335745920:192.168.201.104@tcp:38:kworker.0 00000100:00100000:2.0:1734019665.222153:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335746560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019665.222162:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746048:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019665.222172:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019665.222176:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86dd500 x1818247335746048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019665.222181:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335746048/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019665.222185:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019665.222187:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019665.222189:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746048:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019665.222210:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746176:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019665.222219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019665.222223:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86df800 x1818247335746176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019665.222228:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335746176/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019665.222232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019665.222234:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019665.222235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746176:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019665.222239:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335746304:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019665.222246:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019665.222249:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86de300 x1818247335746304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019665.222254:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335746304/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019665.222258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019665.222259:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019665.222261:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335746304:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019665.222265:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335746432:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019665.222273:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019665.222277:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86dc380 x1818247335746432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019665.222282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335746432/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019665.222286:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019665.222287:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019665.222288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335746432:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019665.222298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335746560:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019665.222305:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019665.222309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019665/real 1734019665] req@ffff8800a86dea00 x1818247335746560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019665.222314:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335746560/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019665.222318:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019665.222319:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019665.222320:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335746560:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019665.295920:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019666.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019666.297922:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019667.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019667.300035:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019668.213821:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019668.301939:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019669.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019669.303891:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019670.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019670.229807:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.229816:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019670.229821:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019670.229834:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.229845:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.229854:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019670.229902:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335746688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019670.229947:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.229951:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019670.229953:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019670.229958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.229961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.229964:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019670.229976:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019670.229986:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335746816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019670.229996:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.230000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019670.230001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019670.230005:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.230007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.230011:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019670.230015:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019670.230023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335746944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019670.230032:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.230035:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019670.230036:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019670.230041:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.230043:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.230047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019670.230051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019670.230063:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335747072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019670.230072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.230075:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019670.230077:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019670.230078:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335746688:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019670.230081:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.230083:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.230087:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019670.230091:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019670.230099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335747200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019670.230166:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.230168:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019670.230169:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019670.230171:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.230184:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.230186:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019670.230189:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019670.230194:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019670.230195:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335747328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019670.230201:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019670.230203:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019670.230203:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019670.230205:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3934 00000100:00080000:2.0:1734019670.230209:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019670.230210:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230212:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86dce00 x1818247335746688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019670.230213:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019670.230217:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335747456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019670.230223:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335746688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230229:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230231:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230234:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335746688:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019670.230238:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746816:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019670.230249:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230252:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86ddf80 x1818247335746816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019670.230258:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335746816/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230264:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746816:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019670.230268:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746944:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019670.230276:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230279:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86dfb80 x1818247335746944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019670.230284:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335746944/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230290:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230292:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335746944:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019670.230308:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747072:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019670.230314:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230318:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86dea00 x1818247335747072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019670.230323:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335747072/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230330:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747072:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019670.230332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335747200:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019670.230339:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86dc380 x1818247335747200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019670.230346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335747200/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230351:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335747200:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019670.230357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335747328:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019670.230363:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230383:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86de300 x1818247335747328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019670.230393:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335747328/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230401:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230403:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230406:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335747328:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019670.230412:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335747456:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019670.230426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019670.230432:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019670/real 1734019670] req@ffff8800a86df800 x1818247335747456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019670.230441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335747456/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019670.230448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019670.230451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019670.230454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335747456:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019670.305918:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019671.213758:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019671.307887:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019672.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019672.309917:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019673.213916:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019673.311979:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019674.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019674.313957:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019675.213823:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019675.237814:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.237823:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019675.237829:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019675.237841:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.237851:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.237861:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019675.237906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335747584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019675.237943:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.237947:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019675.237949:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019675.237954:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.237957:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.237961:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019675.237973:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019675.237984:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335747712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019675.237994:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.237997:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019675.237999:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019675.238003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.238005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.238009:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019675.238013:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019675.238021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335747840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019675.238066:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.238069:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019675.238071:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019675.238075:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.238078:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.238082:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238085:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335747584:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019675.238087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019675.238099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335747968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019675.238109:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.238113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019675.238115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019675.238119:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.238122:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.238125:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019675.238129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019675.238137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335748096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019675.238145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.238148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019675.238149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019675.238153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.238156:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.238160:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019675.238162:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734019675.238164:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019675.238172:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335748224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019675.238180:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019675.238183:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019675.238185:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019675.238187:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86dd500 x1818247335747584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019675.238188:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3939 00000100:00080000:2.0:1734019675.238191:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019675.238194:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019675.238199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019675.238207:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335748352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019675.238208:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335747584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238220:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238223:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238228:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335747584:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019675.238235:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747712:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019675.238252:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238259:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86df100 x1818247335747712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019675.238271:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335747712/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238278:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238281:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747712:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019675.238290:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747840:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019675.238303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238309:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86dc700 x1818247335747840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019675.238321:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335747840/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238329:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238331:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238334:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747840:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019675.238359:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747968:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019675.238372:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238378:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86dca80 x1818247335747968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019675.238388:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335747968/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238396:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238398:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238401:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335747968:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019675.238406:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748096:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019675.238418:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238424:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86dd880 x1818247335748096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019675.238434:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335748096/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238441:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238444:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238448:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748096:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019675.238453:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748224:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019675.238465:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238471:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86df480 x1818247335748224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019675.238480:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335748224/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238488:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238490:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238494:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748224:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019675.238499:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748352:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019675.238511:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019675.238517:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019675/real 1734019675] req@ffff8800a86de680 x1818247335748352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019675.238526:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335748352/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019675.238534:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019675.238536:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019675.238540:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748352:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019675.315939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019676.213841:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019676.317953:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019677.213848:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019677.319878:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019678.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019678.321912:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019679.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019679.323988:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019680.213810:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019680.245765:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.245772:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019680.245776:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019680.245785:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.245793:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.245800:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019680.245839:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335748480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019680.245862:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.245865:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019680.245866:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019680.245869:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.245871:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.245874:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019680.245881:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019680.245888:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335748608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019680.245888:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335748480:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019680.245893:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.245895:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019680.245896:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019680.245899:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.245901:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.245904:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019680.245907:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019680.245912:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335748736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019680.245916:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.245918:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019680.245918:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019680.245921:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.245922:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.245924:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019680.245926:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019680.245934:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335748864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019680.245938:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.245955:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019680.245956:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019680.245958:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.245965:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.245967:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019680.245970:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1734019680.245974:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019680.245977:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335748992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019680.245986:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.245987:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019680.245988:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019680.245991:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.245992:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.245994:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019680.245995:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86ddc00 x1818247335748480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019680.245996:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019680.246001:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335749120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019680.246005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019680.246007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019680.246008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019680.246008:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335748480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019680.246010:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3944 00000100:00080000:2.0:1734019680.246011:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019680.246013:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00080000:3.0:1734019680.246014:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1734019680.246016:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1734019680.246016:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246019:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335748480:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019680.246021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335749248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019680.246036:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335748608:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019680.246048:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019680.246052:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86dc000 x1818247335748608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019680.246059:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335748608/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019680.246062:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019680.246064:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246065:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335748608:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019680.246069:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335748736:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019680.246077:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019680.246080:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86dd180 x1818247335748736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019680.246087:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335748736/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019680.246091:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019680.246092:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246094:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335748736:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019680.246096:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335748864:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019680.246103:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019680.246106:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86de680 x1818247335748864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019680.246111:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335748864/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019680.246115:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019680.246116:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246117:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335748864:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019680.246120:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748992:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019680.246128:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019680.246131:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86df480 x1818247335748992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019680.246136:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335748992/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019680.246145:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019680.246146:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246148:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335748992:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019680.246151:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335749120:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019680.246157:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019680.246160:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86dd880 x1818247335749120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019680.246165:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335749120/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019680.246169:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019680.246170:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246172:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335749120:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019680.246175:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335749248:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019680.246182:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019680.246186:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019680/real 1734019680] req@ffff8800a86dca80 x1818247335749248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019680.246190:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335749248/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019680.246194:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019680.246195:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019680.246197:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335749248:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019680.325846:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019681.214012:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019681.328023:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019682.213804:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019682.329994:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019683.213813:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019683.331980:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019684.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019684.333949:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019685.213851:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019685.253851:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.253890:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019685.253895:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.253908:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00080000:2.0:1734019685.253918:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.253928:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019685.253974:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335749376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019685.254011:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.254015:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019685.254017:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.254022:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00080000:2.0:1734019685.254025:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.254028:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019685.254040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019685.254051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335749504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019685.254061:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.254065:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019685.254066:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.254070:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00080000:2.0:1734019685.254073:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.254077:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019685.254081:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019685.254090:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335749632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019685.254100:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.254103:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019685.254105:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.254109:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00080000:2.0:1734019685.254111:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.254114:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019685.254119:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019685.254122:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335749376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019685.254129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335749760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019685.254138:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.254142:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019685.254143:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.254147:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00080000:2.0:1734019685.254150:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.254153:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019685.254157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019685.254165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335749888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019685.254174:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.254177:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019685.254179:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.254183:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00080000:2.0:1734019685.254195:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.254199:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019685.254203:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019685.254212:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335750016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019685.254221:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019685.254224:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019685.254226:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019685.254230:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3949 00000100:00100000:3.0:1734019685.254231:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335749504:192.168.201.104@tcp:38:kworker.0 00000100:00080000:2.0:1734019685.254232:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019685.254235:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019685.254240:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019685.254247:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335750144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019685.254253:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335749632:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019685.254279:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335749760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019685.254295:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335749888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019685.254311:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019685.254326:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750144:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019685.335922:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019686.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019686.337960:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019687.213869:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019687.339888:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019688.213863:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000400:00000100:1.0:1734019688.259985:0:1959:0:(lib-socket.c:428:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.104:988 00000400:00000100:1.0:1734019688.260034:0:1959:0:(acceptor.c:84:lnet_connect_console_error()) Connection to 192.168.201.104@tcp at host 192.168.201.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1734019688.260053:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019688.260090:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019688.260099:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019688.260107:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019688.260113:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019688.260120:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000800:00000100:1.0:1734019688.260126:0:1959:0:(socklnd_cb.c:423:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.104@tcp 00000100:00100000:3.0:1734019688.260165:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86df800 x1818247335749760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260184:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335749760/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260193:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260196:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260200:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335749760:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019688.260206:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86de300 x1818247335749888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260213:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335749888/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260217:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260220:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260221:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335749888:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019688.260225:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86dc380 x1818247335750016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260231:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335750016/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260236:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260237:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260239:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750016:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019688.260243:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86dea00 x1818247335750144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260249:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335750144/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260252:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260255:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750144:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019688.260263:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86dc700 x1818247335749376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260270:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335749376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260273:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260275:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260277:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335749376:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019688.260280:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86df100 x1818247335749504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260285:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335749504/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260289:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260291:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260293:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335749504:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019688.260296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019685/real 1734019688] req@ffff8800a86dd500 x1818247335749632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019688.260302:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335749632/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019688.260306:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019688.260308:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019688.260309:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335749632:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019688.341875:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019689.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019689.343918:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019690.213866:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019690.261843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.261853:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019690.261858:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.261871:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3954 00000100:00080000:2.0:1734019690.261883:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.261893:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019690.261942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335750272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019690.261981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.261985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019690.261987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.261991:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3954 00000100:00080000:2.0:1734019690.261994:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.261998:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019690.262010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019690.262021:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335750400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019690.262031:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.262034:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019690.262035:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.262040:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3954 00000100:00080000:2.0:1734019690.262042:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.262046:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019690.262051:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019690.262059:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335750528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019690.262067:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.262071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019690.262072:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.262077:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3954 00000100:00080000:2.0:1734019690.262079:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.262083:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019690.262087:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019690.262099:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335750656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019690.262103:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335750272:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019690.262107:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.262111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019690.262112:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.262116:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3954 00000100:00080000:2.0:1734019690.262119:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.262122:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019690.262127:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019690.262136:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335750784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019690.262145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.262148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019690.262149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.262153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3954 00000100:00080000:2.0:1734019690.262156:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.262159:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019690.262163:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019690.262171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335750912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019690.262180:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019690.262183:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019690.262185:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019690.262189:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3954 00000800:00000100:3.0:1734019690.262189:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019690.262191:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019690.262195:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019690.262199:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019690.262207:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335751040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019690.262217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86dfb80 x1818247335750272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262236:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335750272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262248:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262251:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262256:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335750272:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019690.262263:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335750400:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019690.262282:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019690.262288:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86ddf80 x1818247335750400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262300:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335750400/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262308:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262310:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262314:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335750400:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019690.262320:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335750528:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019690.262334:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019690.262340:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86dce00 x1818247335750528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262351:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335750528/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262359:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262362:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262365:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335750528:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019690.262392:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335750656:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019690.262405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019690.262411:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86dd500 x1818247335750656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262421:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335750656/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262435:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335750656:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019690.262441:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750784:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019690.262452:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019690.262458:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86df100 x1818247335750784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262468:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335750784/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262475:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262478:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262482:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750784:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019690.262488:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750912:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019690.262500:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019690.262505:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86dc700 x1818247335750912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262515:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335750912/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262522:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262525:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262528:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335750912:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019690.262532:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751040:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019690.262545:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019690.262551:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019690/real 1734019690] req@ffff8800a86dea00 x1818247335751040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019690.262560:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335751040/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019690.262568:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019690.262570:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019690.262574:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751040:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019690.345983:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019691.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019691.347950:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019692.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019692.349989:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019693.213845:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019693.351940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019694.213809:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019694.353949:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019695.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019695.269807:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.269816:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019695.269821:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019695.269833:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.269843:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.269852:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019695.269896:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335751168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019695.269931:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.269935:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019695.269938:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019695.269943:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.269946:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.269950:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019695.269961:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019695.269972:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335751296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019695.269981:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.269985:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019695.269987:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019695.269990:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.269993:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.269997:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019695.270001:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019695.270009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335751424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019695.270018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.270021:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019695.270022:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019695.270026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.270028:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.270032:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019695.270036:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019695.270039:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335751168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:2.0:1734019695.270047:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335751552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019695.270057:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.270061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019695.270063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019695.270066:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.270069:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.270072:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019695.270077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019695.270084:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335751680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019695.270092:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.270095:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019695.270096:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019695.270100:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.270112:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.270116:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019695.270118:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734019695.270121:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019695.270129:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335751808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019695.270137:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019695.270140:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019695.270142:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019695.270144:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86dc380 x1818247335751168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019695.270145:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3959 00000100:00080000:2.0:1734019695.270149:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019695.270153:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019695.270157:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1734019695.270164:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335751168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:2.0:1734019695.270165:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335751936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:3.0:1734019695.270175:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270178:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270183:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335751168:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019695.270190:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335751296:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019695.270207:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019695.270214:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86de300 x1818247335751296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019695.270225:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335751296/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019695.270233:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270235:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270238:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335751296:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019695.270244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335751424:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019695.270258:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019695.270265:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86df800 x1818247335751424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019695.270276:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335751424/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019695.270283:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270285:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270288:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335751424:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019695.270315:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335751552:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019695.270328:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019695.270334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86dca80 x1818247335751552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019695.270344:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335751552/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019695.270351:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270354:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270357:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335751552:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019695.270362:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751680:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019695.270374:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019695.270379:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86dd880 x1818247335751680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019695.270389:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335751680/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019695.270404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270410:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751680:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019695.270414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751808:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019695.270426:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019695.270431:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86df480 x1818247335751808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019695.270441:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335751808/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019695.270448:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751808:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019695.270458:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751936:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019695.270470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019695.270476:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019695/real 1734019695] req@ffff8800a86de680 x1818247335751936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019695.270486:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335751936/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019695.270494:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019695.270496:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019695.270499:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335751936:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019695.355899:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019696.213914:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019696.357931:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019697.213808:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019697.359971:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019698.213955:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019698.361940:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019699.213834:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019699.363971:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019700.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019700.277843:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.277852:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019700.277857:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019700.277870:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00080000:2.0:1734019700.277880:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.277890:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019700.277937:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335752064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019700.277974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.277978:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019700.277980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019700.277985:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00080000:2.0:1734019700.277988:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.277992:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019700.278004:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019700.278015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335752192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019700.278033:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.278037:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019700.278038:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019700.278042:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00080000:2.0:1734019700.278044:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.278048:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019700.278052:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019700.278061:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335752320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019700.278070:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.278073:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019700.278075:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019700.278079:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00100000:3.0:1734019700.278079:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335752064:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019700.278082:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.278085:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019700.278089:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019700.278100:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335752448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019700.278108:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.278111:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019700.278113:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019700.278117:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00080000:2.0:1734019700.278120:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.278124:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019700.278129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019700.278137:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335752576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019700.278145:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.278148:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019700.278149:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019700.278153:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00080000:2.0:1734019700.278155:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.278159:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019700.278159:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734019700.278163:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019700.278171:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335752704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019700.278179:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019700.278182:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019700.278183:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019700.278185:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86dd180 x1818247335752064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019700.278187:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3964 00000100:00080000:2.0:1734019700.278190:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019700.278193:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019700.278197:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019700.278205:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335752832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019700.278205:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335752064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278217:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278221:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335752064:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019700.278233:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335752192:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019700.278250:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019700.278257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86dc000 x1818247335752192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019700.278269:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335752192/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278277:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278279:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278283:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335752192:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019700.278312:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335752320:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019700.278327:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019700.278334:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86ddc00 x1818247335752320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019700.278345:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335752320/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278353:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278355:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278358:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335752320:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019700.278363:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335752448:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019700.278375:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019700.278381:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86de680 x1818247335752448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019700.278391:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335752448/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278398:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278401:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278405:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335752448:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019700.278410:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335752576:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019700.278422:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019700.278428:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86df480 x1818247335752576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019700.278440:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335752576/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278447:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278450:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278453:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335752576:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019700.278457:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335752704:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019700.278470:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019700.278475:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86dd880 x1818247335752704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019700.278485:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335752704/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278492:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278494:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278497:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335752704:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019700.278502:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335752832:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019700.278514:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019700.278520:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019700/real 1734019700] req@ffff8800a86dca80 x1818247335752832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019700.278530:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335752832/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019700.278537:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019700.278540:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019700.278543:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335752832:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019700.365880:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019701.213879:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019701.367967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019702.213831:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019702.369974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019703.213852:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019703.371959:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019704.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019704.373986:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019705.213814:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019705.285826:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.285835:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019705.285841:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.285852:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.285862:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.285872:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019705.285917:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335752960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019705.285953:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.285958:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019705.285960:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.285965:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.285968:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.285972:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019705.285983:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019705.285995:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335753088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019705.286004:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.286007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019705.286010:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.286014:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.286017:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.286020:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019705.286025:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019705.286034:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335753216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019705.286042:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.286045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019705.286047:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.286051:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.286053:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.286057:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286059:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335752960:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019705.286061:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019705.286071:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335753344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019705.286080:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.286083:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019705.286085:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.286089:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.286091:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.286095:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019705.286099:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019705.286107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335753472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019705.286116:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.286119:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019705.286121:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.286124:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.286128:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.286132:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019705.286136:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019705.286140:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019705.286144:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335753600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019705.286151:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019705.286154:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019705.286156:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019705.286159:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3969 00000100:00080000:2.0:1734019705.286162:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019705.286166:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286166:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86df800 x1818247335752960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019705.286170:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019705.286178:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335753728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019705.286187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335752960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286199:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286203:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286208:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335752960:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019705.286215:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753088:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019705.286232:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286239:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86de300 x1818247335753088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019705.286250:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335753088/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286258:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286261:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753088:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019705.286271:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753216:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019705.286286:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286292:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86dc380 x1818247335753216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019705.286303:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335753216/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286311:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286314:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286317:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753216:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019705.286345:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753344:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019705.286358:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286364:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86dea00 x1818247335753344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019705.286375:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335753344/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286383:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286385:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286388:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753344:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019705.286393:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335753472:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019705.286405:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286412:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86dc700 x1818247335753472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019705.286422:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335753472/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286429:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286431:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286434:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335753472:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019705.286440:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335753600:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019705.286451:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286457:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86df100 x1818247335753600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019705.286467:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335753600/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286474:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286476:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286479:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335753600:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019705.286484:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335753728:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019705.286496:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019705.286502:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019705/real 1734019705] req@ffff8800a86dd500 x1818247335753728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019705.286511:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335753728/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019705.286574:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019705.286576:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019705.286580:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335753728:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019705.375974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019706.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019706.377945:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019707.213839:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019707.379954:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019708.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019708.381950:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019709.213843:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019709.383928:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019710.213865:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019710.293856:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.293867:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019710.293872:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.293883:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.293893:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.293900:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019710.293938:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335753856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019710.293967:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.293969:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019710.293971:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.293974:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.293976:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.293978:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019710.293986:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019710.293993:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335753984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019710.293998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.294000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019710.294001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.294003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.294005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.294007:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019710.294010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019710.294015:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335754112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019710.294020:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.294023:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019710.294023:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.294026:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.294027:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.294029:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019710.294031:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019710.294039:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335754240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019710.294043:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.294045:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019710.294046:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.294049:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.294050:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.294052:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294053:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335753856:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019710.294055:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019710.294060:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335754368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019710.294065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.294067:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019710.294068:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.294071:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.294072:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.294075:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019710.294077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019710.294082:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335754496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019710.294086:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019710.294088:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019710.294088:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019710.294091:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3974 00000100:00080000:2.0:1734019710.294092:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019710.294094:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019710.294096:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019710.294101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335754624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019710.294117:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294138:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86dce00 x1818247335753856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294152:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335753856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294159:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294161:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294164:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335753856:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019710.294167:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753984:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019710.294178:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294181:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86ddf80 x1818247335753984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294187:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335753984/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294196:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294198:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294200:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335753984:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019710.294203:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335754112:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019710.294211:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294214:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86dfb80 x1818247335754112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294219:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335754112/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294223:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294224:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294226:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335754112:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019710.294246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335754240:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019710.294253:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86dd500 x1818247335754240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294262:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335754240/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294267:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294269:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335754240:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019710.294273:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335754368:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019710.294280:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294284:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86df100 x1818247335754368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294289:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335754368/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294292:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294293:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294295:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335754368:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019710.294298:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335754496:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019710.294303:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294306:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86dc700 x1818247335754496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294311:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335754496/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294316:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294318:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294319:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335754496:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019710.294322:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335754624:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019710.294329:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019710.294332:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019710/real 1734019710] req@ffff8800a86dea00 x1818247335754624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019710.294338:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335754624/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019710.294342:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019710.294343:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019710.294344:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335754624:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019710.386007:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019711.213752:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019711.387982:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019712.213836:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019712.389856:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019713.213854:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019713.392001:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019714.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019714.393953:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019715.213857:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019715.301849:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.301858:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019715.301863:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.301875:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.301885:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019715.301895:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019715.301941:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335754752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019715.301977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.301981:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019715.301983:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.301988:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.301991:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019715.301995:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019715.302008:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019715.302019:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335754880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019715.302029:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.302032:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019715.302034:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.302038:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.302041:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019715.302044:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019715.302049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019715.302057:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335755008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019715.302065:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.302068:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019715.302070:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.302075:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.302077:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019715.302081:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302082:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335754752:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019715.302086:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019715.302095:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335755136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019715.302104:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.302107:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019715.302109:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.302112:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.302115:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019715.302118:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019715.302123:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019715.302131:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335755264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019715.302140:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.302143:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019715.302144:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.302148:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.302151:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019715.302155:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019715.302159:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019715.302163:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019715.302167:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335755392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019715.302176:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019715.302179:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019715.302181:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019715.302184:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3979 00000100:00080000:2.0:1734019715.302187:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019715.302190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86dc380 x1818247335754752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019715.302191:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019715.302195:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019715.302203:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335755520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019715.302210:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335754752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302221:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302225:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302230:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335754752:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019715.302237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335754880:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019715.302255:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302262:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86de300 x1818247335754880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019715.302282:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335754880/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302290:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302293:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302296:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335754880:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019715.302302:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755008:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019715.302316:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302322:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86df800 x1818247335755008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019715.302332:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335755008/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302339:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302341:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302345:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755008:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019715.302415:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755136:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019715.302433:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302440:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86dca80 x1818247335755136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019715.302451:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335755136/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302459:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302462:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302465:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755136:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019715.302471:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335755264:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019715.302484:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302490:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86dd880 x1818247335755264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019715.302500:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335755264/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302508:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302510:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302513:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335755264:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019715.302518:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335755392:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019715.302530:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302536:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86df480 x1818247335755392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019715.302545:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335755392/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302553:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302556:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302559:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335755392:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019715.302565:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335755520:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019715.302581:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019715.302587:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019715/real 1734019715] req@ffff8800a86de680 x1818247335755520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019715.302598:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335755520/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019715.302605:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019715.302607:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019715.302610:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335755520:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019715.395963:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019716.213913:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019716.397967:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019717.213835:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019717.399973:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019718.213847:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019718.401975:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019719.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019719.403962:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019720.213876:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019720.309859:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.309870:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019720.309875:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.309885:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.309894:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019720.309902:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019720.309942:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335755648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019720.309971:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.309974:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019720.309975:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.309979:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.309980:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019720.309982:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019720.309992:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019720.309999:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335755776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019720.310005:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.310007:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019720.310008:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.310011:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.310013:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019720.310015:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019720.310018:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019720.310023:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335755904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019720.310027:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.310029:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019720.310030:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.310032:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.310033:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019720.310035:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019720.310037:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019720.310045:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335756032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019720.310050:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.310052:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019720.310053:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.310056:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.310057:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019720.310058:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335755648:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019720.310060:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019720.310062:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019720.310068:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335756160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019720.310072:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.310074:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019720.310074:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.310077:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.310078:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019720.310080:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019720.310082:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019720.310087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335756288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019720.310091:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019720.310092:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019720.310093:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019720.310095:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3984 00000100:00080000:2.0:1734019720.310097:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019720.310099:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019720.310102:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019720.310107:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335756416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019720.310123:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310143:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86ddc00 x1818247335755648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310157:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335755648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310164:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310167:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310169:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335755648:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019720.310173:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755776:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019720.310186:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310190:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86dc000 x1818247335755776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310196:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335755776/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310200:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310202:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310203:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755776:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019720.310206:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755904:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019720.310214:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310217:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86dd180 x1818247335755904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310222:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335755904/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310226:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310227:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310229:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335755904:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019720.310246:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756032:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019720.310254:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310257:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86de680 x1818247335756032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310293:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335756032/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310298:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310300:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310302:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756032:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019720.310305:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335756160:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019720.310313:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310316:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86df480 x1818247335756160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335756160/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310327:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310329:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335756160:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019720.310332:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335756288:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019720.310338:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310341:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86dd880 x1818247335756288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310346:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335756288/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310350:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310352:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310353:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335756288:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019720.310357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335756416:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019720.310364:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019720.310367:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019720/real 1734019720] req@ffff8800a86dca80 x1818247335756416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019720.310372:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335756416/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019720.310376:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019720.310378:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019720.310380:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335756416:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019720.405974:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019721.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019721.407882:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019722.213853:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019722.409964:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019723.213822:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019723.411920:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019724.213953:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019724.413947:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019725.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019725.317836:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.317846:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019725.317852:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.317864:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.317874:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.317884:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019725.317931:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335756544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019725.317967:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.317971:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019725.317973:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.317978:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.317981:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.317985:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019725.317997:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019725.318009:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335756672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019725.318018:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.318022:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019725.318024:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.318028:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.318031:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.318034:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019725.318040:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019725.318049:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335756800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019725.318058:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.318061:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019725.318063:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.318066:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.318069:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.318072:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318076:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335756544:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019725.318077:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019725.318087:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335756928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019725.318096:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.318100:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019725.318101:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.318105:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.318107:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.318111:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019725.318115:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019725.318123:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335757056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019725.318131:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.318134:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019725.318137:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.318141:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.318153:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.318157:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000800:00000100:3.0:1734019725.318161:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000008:00100000:2.0:1734019725.318162:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019725.318170:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335757184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019725.318178:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019725.318214:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019725.318215:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019725.318220:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3989 00000100:00080000:2.0:1734019725.318223:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019725.318228:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86df800 x1818247335756544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000008:00100000:2.0:1734019725.318233:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019725.318243:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335757312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019725.318251:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335756544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318263:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318267:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318272:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335756544:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019725.318280:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756672:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019725.318302:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318310:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86de300 x1818247335756672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019725.318322:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335756672/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318330:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318332:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318335:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756672:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019725.318341:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756800:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019725.318356:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318363:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86dc380 x1818247335756800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019725.318373:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335756800/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318381:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318383:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318386:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756800:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019725.318414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756928:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019725.318432:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318439:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86dea00 x1818247335756928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019725.318450:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335756928/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318458:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318460:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318463:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335756928:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019725.318468:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757056:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019725.318480:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318485:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86dc700 x1818247335757056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019725.318495:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335757056/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318503:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318505:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318508:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757056:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019725.318514:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757184:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019725.318525:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318531:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86df100 x1818247335757184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019725.318548:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335757184/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318556:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318558:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318561:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757184:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019725.318566:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757312:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019725.318578:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019725.318584:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019725/real 1734019725] req@ffff8800a86dd500 x1818247335757312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019725.318594:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335757312/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019725.318601:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019725.318603:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019725.318606:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757312:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019725.415955:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019726.213840:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019726.417940:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019727.213798:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019727.420002:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019728.213849:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019728.421959:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019729.213815:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019729.423929:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019730.213773:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019730.325782:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325792:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019730.325796:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325804:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00080000:2.0:1734019730.325812:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.325819:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019730.325852:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dfb80 x1818247335757440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019730.325881:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325884:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019730.325885:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325888:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00080000:2.0:1734019730.325890:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.325892:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019730.325899:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019730.325906:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddf80 x1818247335757568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019730.325911:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325913:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019730.325914:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325916:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00080000:2.0:1734019730.325918:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.325920:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019730.325922:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019730.325927:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dce00 x1818247335757696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019730.325932:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325933:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019730.325934:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325936:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00080000:2.0:1734019730.325938:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.325940:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019730.325942:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019730.325950:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd500 x1818247335757824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019730.325954:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325956:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019730.325957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325959:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00100000:3.0:1734019730.325959:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335757440:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019730.325961:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.325963:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019730.325965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019730.325970:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df100 x1818247335757952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019730.325974:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325976:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019730.325977:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325979:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00080000:2.0:1734019730.325981:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.325983:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019730.325985:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019730.325990:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc700 x1818247335758080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019730.325994:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019730.325996:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019730.325997:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019730.325999:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3994 00000100:00080000:2.0:1734019730.326007:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019730.326009:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019730.326012:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1734019730.326015:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:2.0:1734019730.326017:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dea00 x1818247335758208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019730.326033:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86dfb80 x1818247335757440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326046:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dfb80 x1818247335757440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326052:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326054:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326057:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335757440:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019730.326061:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335757568:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019730.326071:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019730.326075:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86ddf80 x1818247335757568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326081:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddf80 x1818247335757568/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326085:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326086:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326088:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335757568:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019730.326091:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335757696:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019730.326131:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019730.326135:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86dce00 x1818247335757696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326141:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dce00 x1818247335757696/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326145:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326147:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326148:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335757696:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019730.326167:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335757824:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019730.326176:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019730.326179:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86dd500 x1818247335757824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326185:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd500 x1818247335757824/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326189:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326190:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326192:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335757824:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019730.326194:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757952:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019730.326201:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019730.326204:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86df100 x1818247335757952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326209:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df100 x1818247335757952/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326213:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326214:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326216:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335757952:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019730.326219:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758080:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019730.326227:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019730.326230:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86dc700 x1818247335758080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326235:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc700 x1818247335758080/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326239:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326240:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326242:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758080:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019730.326244:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758208:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019730.326251:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019730.326254:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019730/real 1734019730] req@ffff8800a86dea00 x1818247335758208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019730.326259:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dea00 x1818247335758208/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019730.326262:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019730.326263:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019730.326265:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758208:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019730.425852:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019731.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019731.427891:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019732.213770:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019732.429960:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019733.213837:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019733.431909:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019734.213921:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:0.0:1734019734.433846:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019735.213811:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019735.333793:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.333804:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019735.333808:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.333818:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.333826:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.333833:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019735.333872:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc380 x1818247335758336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019735.333900:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.333902:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019735.333904:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.333907:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.333909:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.333911:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019735.333921:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019735.333928:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de300 x1818247335758464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019735.333933:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.333936:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019735.333937:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.333939:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.333940:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.333942:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019735.333945:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019735.333951:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df800 x1818247335758592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019735.333955:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.333957:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019735.333957:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.333960:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.333961:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.333963:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019735.333965:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019735.333972:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335758720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019735.333977:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.333979:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019735.333980:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.333983:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.333984:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.333986:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1734019735.333988:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335758336:192.168.201.104@tcp:250:kworker.0 00000008:00100000:2.0:1734019735.333989:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019735.333994:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335758848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019735.333998:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.334000:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019735.334001:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.334003:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.334005:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.334008:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019735.334010:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019735.334035:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335758976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019735.334039:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019735.334041:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019735.334042:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019735.334044:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 3999 00000100:00080000:2.0:1734019735.334045:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019735.334047:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019735.334049:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019735.334054:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335759104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000800:00000100:3.0:1734019735.334083:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334104:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86dc380 x1818247335758336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334118:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc380 x1818247335758336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334125:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334127:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334129:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335758336:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019735.334133:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335758464:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019735.334145:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334148:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86de300 x1818247335758464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334154:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de300 x1818247335758464/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334158:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334160:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334161:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335758464:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019735.334164:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335758592:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019735.334173:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334176:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86df800 x1818247335758592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334181:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df800 x1818247335758592/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334189:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334191:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334193:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335758592:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019735.334211:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335758720:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019735.334219:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334222:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86dca80 x1818247335758720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334227:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335758720/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334232:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334233:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334235:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335758720:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019735.334237:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758848:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019735.334244:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334246:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86dd880 x1818247335758848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334251:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335758848/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334255:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334256:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334258:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758848:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019735.334262:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758976:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019735.334269:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334272:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86df480 x1818247335758976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334277:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335758976/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334281:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334282:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334284:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335758976:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019735.334287:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335759104:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019735.334293:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019735.334296:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019735/real 1734019735] req@ffff8800a86de680 x1818247335759104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019735.334301:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335759104/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019735.334305:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019735.334306:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019735.334307:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335759104:192.168.201.104@tcp:8:kworker.0 00010000:00010000:3.0:1734019735.435965:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1734019736.213859:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019736.437960:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019737.213931:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:2.0:1734019737.439961:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019738.213927:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019738.441939:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019739.213830:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019739.444023:0:10899:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019740.213844:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00000100:00080000:2.0:1734019740.341828:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) a5fb1fbe-7587-48b8-9da9-5c701ca16068->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.341839:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1734019740.341844:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019740.341858:0:11946:0:(import.c:523:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.341868:0:11946:0:(import.c:580:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019740.341878:0:11946:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import ffff8800b588a000 using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1734019740.341925:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd180 x1818247335759232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019740.342007:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.342011:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019740.342013:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019740.342019:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.342022:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019740.342027:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800b588c800: import ffff8800aa649800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019740.342038:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019740.342051:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dc000 x1818247335759360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019740.342068:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.342071:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019740.342073:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019740.342078:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.342083:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019740.342087:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800b588c800: import ffff8800aa64a800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019740.342092:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 28672, lost: 0. 00000100:00100000:2.0:1734019740.342101:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86ddc00 x1818247335759488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019740.342110:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 1c97ffb5-66ce-4359-8d50-0488c17e03ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.342113:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019740.342115:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019740.342119:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.342121:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1734019740.342123:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335759232:192.168.201.104@tcp:250:kworker.0 00000100:00080000:2.0:1734019740.342124:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800b588c800: import ffff8800aa64c000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019740.342129:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019740.342140:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86de680 x1818247335759616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019740.342149:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.342152:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1734019740.342154:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019740.342158:0:11946:0:(import.c:523:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.342161:0:11946:0:(import.c:580:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019740.342165:0:11946:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff8800aa629800: import ffff8800a98c0800 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019740.342169:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34603008 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1734019740.342177:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86df480 x1818247335759744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019740.342192:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.342195:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1734019740.342196:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1734019740.342200:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.342203:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1734019740.342205:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00080000:2.0:1734019740.342207:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff8800aa629800: import ffff8800a98c1000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019740.342211:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019740.342218:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dd880 x1818247335759872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00080000:2.0:1734019740.342226:0:11946:0:(pinger.c:254:ptlrpc_pinger_process_import()) 6ad7014d-1575-48d3-b7ba-eccec1d268ba->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1734019740.342229:0:11946:0:(pinger.c:262:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1734019740.342231:0:11946:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1734019740.342232:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86dd180 x1818247335759232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:2.0:1734019740.342234:0:11946:0:(import.c:523:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: connect to NID 192.168.201.104@tcp last attempt 4004 00000100:00080000:2.0:1734019740.342237:0:11946:0:(import.c:580:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1734019740.342240:0:11946:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff8800aa629800: import ffff8800a98c7000 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1734019740.342244:0:11946:0:(osc_request.c:3668:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34603008 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1734019740.342252:0:11946:0:(import.c:814:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8800a86dca80 x1818247335760000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 00000100:00100000:3.0:1734019740.342253:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd180 x1818247335759232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342264:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800b588a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342268:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342273:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a5fb1fbe-7587-48b8-9da9-5c701ca16068:1968:1818247335759232:192.168.201.104@tcp:250:kworker.0 00000100:00100000:3.0:1734019740.342281:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335759360:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019740.342298:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019740.342305:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86dc000 x1818247335759360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019740.342316:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dc000 x1818247335759360/t0(0) o38->lustre-MDT0000-mdc-ffff8800b588c800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342324:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa649800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342328:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342331:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335759360:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019740.342357:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335759488:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019740.342371:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019740.342377:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86ddc00 x1818247335759488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019740.342396:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86ddc00 x1818247335759488/t0(0) o8->lustre-OST0000-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342404:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64a800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342406:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342409:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335759488:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019740.342414:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335759616:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019740.342427:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019740.342433:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86de680 x1818247335759616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019740.342442:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86de680 x1818247335759616/t0(0) o8->lustre-OST0001-osc-ffff8800b588c800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342449:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800aa64c000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342451:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342454:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1c97ffb5-66ce-4359-8d50-0488c17e03ca:1968:1818247335759616:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019740.342459:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335759744:192.168.201.104@tcp:38:kworker.0 00000800:00000100:3.0:1734019740.342471:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019740.342478:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86df480 x1818247335759744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019740.342488:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86df480 x1818247335759744/t0(0) o38->lustre-MDT0000-mdc-ffff8800aa629800@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342495:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c0800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342499:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342502:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335759744:192.168.201.104@tcp:38:kworker.0 00000100:00100000:3.0:1734019740.342507:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335759872:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019740.342520:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019740.342526:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86dd880 x1818247335759872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019740.342535:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dd880 x1818247335759872/t0(0) o8->lustre-OST0000-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342543:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342545:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342548:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335759872:192.168.201.104@tcp:8:kworker.0 00000100:00100000:3.0:1734019740.342552:0:1968:0:(client.c:1796:ptlrpc_send_new_req()) Sending RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335760000:192.168.201.104@tcp:8:kworker.0 00000800:00000100:3.0:1734019740.342565:0:1968:0:(socklnd_cb.c:975:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.104@tcp 00000100:00100000:3.0:1734019740.342571:0:1968:0:(client.c:2363:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1734019740/real 1734019740] req@ffff8800a86dca80 x1818247335760000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00100000:3.0:1734019740.342580:0:1968:0:(client.c:2394:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8800a86dca80 x1818247335760000/t0(0) o8->lustre-OST0001-osc-ffff8800aa629800@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1734019765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 00000100:00080000:3.0:1734019740.342588:0:1968:0:(import.c:86:import_set_state_nolock()) ffff8800a98c7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1734019740.342590:0:1968:0:(import.c:1479:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1734019740.342594:0:1968:0:(client.c:2281:ptlrpc_check_set()) Completed RPC req@ffff8800a86dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6ad7014d-1575-48d3-b7ba-eccec1d268ba:1968:1818247335760000:192.168.201.104@tcp:8:kworker.0 00010000:00010000:2.0:1734019740.445965:0:10898:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1734019741.213842:0:5236:0:(client.c:2562:ptlrpc_set_wait()) set ffff8800a8602900 going to sleep for 0 seconds 00010000:00010000:3.0:1734019741.447952:0:4738:0:(ldlm_lockd.c:2841:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 77205 lines, 77205 kept, 0 dropped, 0 bad.